diff --git a/.github/workflows/cd-staging.yml b/.github/workflows/cd-staging.yml new file mode 100644 index 0000000..c741b49 --- /dev/null +++ b/.github/workflows/cd-staging.yml @@ -0,0 +1,402 @@ +name: CD Staging + +# Builds, deploys, and tears down the persistent staging stack and per-PR +# preview environments. See `openspec/changes/staging-environments/` for the +# design (decisions on shared Postgres, port allocation, per-PR Caddyfile +# snippets) and CLAUDE.md "Staging & Previews" for the operator-facing view. + +on: + push: + branches: [main] + paths: + - "apps/**" + - "packages/**" + - "pnpm-lock.yaml" + pull_request: + types: [opened, synchronize, reopened, closed] + paths: + - "apps/**" + - "packages/**" + - "pnpm-lock.yaml" + workflow_dispatch: {} + +# Per-target concurrency: persistent staging deploys serialize against +# themselves, each PR's preview lifecycle serializes against itself. +concurrency: + group: staging-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.number) || 'main' }} + cancel-in-progress: true + +jobs: + # ── Build ───────────────────────────────────────────────────────────── + # Tags: + # main push → :staging + : + # PR open/sync/reopen → :pr- + :pr-- + # Skipped entirely on PR close (teardown doesn't need new images). + build-images: + name: Build & Push Docker Images + if: github.event_name != 'pull_request' || github.event.action != 'closed' + runs-on: ubuntu-latest + environment: production + permissions: + contents: read + packages: write + strategy: + matrix: + app: [journal, planner] + outputs: + tag_primary: ${{ steps.tags.outputs.primary }} + tag_sha: ${{ steps.tags.outputs.sha }} + steps: + - uses: actions/checkout@v6 + + - id: tags + name: Compute image tags + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + PRIMARY="pr-${{ github.event.number }}" + SHA="pr-${{ github.event.number }}-${{ github.event.pull_request.head.sha }}" + else + PRIMARY="staging" + SHA="${{ github.sha }}" + fi + echo "primary=$PRIMARY" >> "$GITHUB_OUTPUT" + echo "sha=$SHA" >> "$GITHUB_OUTPUT" + + - uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Decrypt Sentry auth token + run: | + curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64 + chmod +x sops-v3.9.4.linux.amd64 + SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > /tmp/secrets.env + grep SENTRY_AUTH_TOKEN /tmp/secrets.env | cut -d= -f2- | tr -d '\n' > /tmp/sentry_token + + - uses: docker/build-push-action@v7 + with: + context: . + file: apps/${{ matrix.app }}/Dockerfile + push: true + tags: | + ghcr.io/trails-cool/${{ matrix.app }}:${{ steps.tags.outputs.primary }} + ghcr.io/trails-cool/${{ matrix.app }}:${{ steps.tags.outputs.sha }} + build-args: | + SENTRY_RELEASE=${{ steps.tags.outputs.sha }} + secrets: | + SENTRY_AUTH_TOKEN=/tmp/sentry_token + + # ── Deploy persistent staging (main push) ──────────────────────────── + deploy-staging: + name: Deploy Staging + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: [build-images] + runs-on: ubuntu-latest + environment: production + steps: + - uses: actions/checkout@v6 + + - name: Decrypt secrets + run: | + curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64 + chmod +x sops-v3.9.4.linux.amd64 + SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > infrastructure/staging.env + { + echo "DOMAIN=staging.trails.cool" + echo "STAGING_DATABASE=trails_staging" + echo "JOURNAL_HOST_PORT=3100" + echo "PLANNER_HOST_PORT=3101" + echo "JOURNAL_IMAGE_TAG=staging" + echo "PLANNER_IMAGE_TAG=staging" + echo "SENTRY_RELEASE=${{ github.sha }}" + } >> infrastructure/staging.env + + - name: Copy compose file + env to server + uses: appleboy/scp-action@v1 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: root + key: ${{ secrets.DEPLOY_SSH_KEY }} + source: "infrastructure/docker-compose.staging.yml,infrastructure/staging.env" + target: /opt/trails-cool + strip_components: 1 + + - name: Deploy via SSH + uses: appleboy/ssh-action@v1 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: root + key: ${{ secrets.DEPLOY_SSH_KEY }} + script: | + set -euo pipefail + cd /opt/trails-cool + + GHCR_TOKEN=$(grep DEPLOY_GHCR_TOKEN staging.env | cut -d= -f2-) + echo "$GHCR_TOKEN" | docker login ghcr.io -u stigi --password-stdin + + # Bootstrap the shared network so cd-staging works regardless of + # whether cd-infra has already run with the new docker-compose.yml. + # Once cd-infra runs, postgres is permanently joined via compose; + # until then, attach it imperatively. + docker network inspect trails-shared >/dev/null 2>&1 || docker network create trails-shared + PG_CONTAINER=$(docker ps --filter "name=trails-cool-postgres" --format '{{.Names}}' | head -1) + if [ -n "$PG_CONTAINER" ]; then + docker network connect trails-shared "$PG_CONTAINER" 2>/dev/null || true + fi + + # Ensure trails_staging database exists with postgis. Production + # init scripts only run on first data-dir init, so a freshly + # created database has no extensions. + docker compose exec -T postgres psql -U trails -d postgres -tAc \ + "SELECT 1 FROM pg_database WHERE datname='trails_staging'" \ + | grep -q 1 \ + || docker compose exec -T postgres createdb -U trails trails_staging + docker compose exec -T postgres psql -U trails -d trails_staging -c \ + "CREATE EXTENSION IF NOT EXISTS postgis" + + # Pull and deploy staging containers (journal + planner via "persistent" profile) + docker compose -f docker-compose.staging.yml -p trails-staging --env-file staging.env --profile persistent pull + docker compose -f docker-compose.staging.yml -p trails-staging --env-file staging.env --profile persistent run --rm journal npx drizzle-kit push --config /app/packages/db/drizzle.config.ts --force + docker compose -f docker-compose.staging.yml -p trails-staging --env-file staging.env --profile persistent up -d --remove-orphans + + # Reload Caddy so new staging routes (or Caddyfile changes shipped + # via cd-infra) are live. Idempotent. + docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile || true + + docker compose -f docker-compose.staging.yml -p trails-staging ps + + # ── PR preview deploy ──────────────────────────────────────────────── + deploy-preview: + name: Deploy PR Preview + if: github.event_name == 'pull_request' && github.event.action != 'closed' + needs: [build-images] + runs-on: ubuntu-latest + environment: production + permissions: + pull-requests: write + steps: + - uses: actions/checkout@v6 + + - id: ports + name: Compute preview ports + project name + run: | + PR=${{ github.event.number }} + # journal = 3200 + 2N, planner unused (PR previews are journal-only) + JOURNAL_PORT=$((3200 + 2 * PR)) + PLANNER_PORT=$((3201 + 2 * PR)) + echo "pr=$PR" >> "$GITHUB_OUTPUT" + echo "journal_port=$JOURNAL_PORT" >> "$GITHUB_OUTPUT" + echo "planner_port=$PLANNER_PORT" >> "$GITHUB_OUTPUT" + echo "host=pr-$PR.staging.trails.cool" >> "$GITHUB_OUTPUT" + echo "project=trails-pr-$PR" >> "$GITHUB_OUTPUT" + echo "database=trails_pr_$PR" >> "$GITHUB_OUTPUT" + + - name: Decrypt secrets + assemble env + run: | + curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64 + chmod +x sops-v3.9.4.linux.amd64 + SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > infrastructure/staging.env + { + echo "DOMAIN=${{ steps.ports.outputs.host }}" + echo "STAGING_DATABASE=${{ steps.ports.outputs.database }}" + echo "JOURNAL_HOST_PORT=${{ steps.ports.outputs.journal_port }}" + echo "PLANNER_HOST_PORT=${{ steps.ports.outputs.planner_port }}" + echo "JOURNAL_IMAGE_TAG=pr-${{ steps.ports.outputs.pr }}" + echo "PLANNER_IMAGE_TAG=pr-${{ steps.ports.outputs.pr }}" + # PR-preview journals all share the persistent staging planner. + echo "PLANNER_URL=https://planner.staging.trails.cool" + echo "SENTRY_RELEASE=${{ github.event.pull_request.head.sha }}" + } >> infrastructure/staging.env + + - name: Generate per-PR Caddyfile snippet + run: | + mkdir -p infrastructure/sites + cat > infrastructure/sites/pr-${{ steps.ports.outputs.pr }}.caddyfile </dev/null 2>&1 || docker network create trails-shared + PG_CONTAINER=$(docker ps --filter "name=trails-cool-postgres" --format '{{.Names}}' | head -1) + if [ -n "$PG_CONTAINER" ]; then + docker network connect trails-shared "$PG_CONTAINER" 2>/dev/null || true + fi + + # Concurrent preview limit (max 3): if we're at the cap and this + # PR isn't already running, evict the oldest preview project. + ACTIVE=$(docker compose ls --format json --filter "name=trails-pr-" | python3 -c 'import json,sys; data=json.load(sys.stdin); print("\n".join(d["Name"] for d in data))' 2>/dev/null || true) + if [ -n "$ACTIVE" ] && ! echo "$ACTIVE" | grep -qx "$PROJECT"; then + COUNT=$(echo "$ACTIVE" | grep -c '^trails-pr-' || true) + if [ "$COUNT" -ge 3 ]; then + # Pick the oldest by container CreatedAt of any service in the project. + OLDEST=$(docker ps -a --filter "name=trails-pr-" --format '{{.Names}} {{.CreatedAt}}' \ + | awk '{ split($1,a,"-"); print "trails-pr-"a[3], $2" "$3" "$4 }' \ + | sort -k2 \ + | head -1 \ + | awk '{print $1}') + if [ -n "$OLDEST" ] && [ "$OLDEST" != "$PROJECT" ]; then + echo "At cap; evicting oldest preview: $OLDEST" + OLD_PR=${OLDEST#trails-pr-} + docker compose -f docker-compose.staging.yml -p "$OLDEST" --env-file staging.env down --remove-orphans || true + docker compose exec -T postgres dropdb -U trails --if-exists "trails_pr_$OLD_PR" || true + rm -f "sites/pr-$OLD_PR.caddyfile" + fi + fi + fi + + # Ensure per-PR database exists with postgis. (See deploy-staging + # for why we have to enable the extension explicitly.) + docker compose exec -T postgres psql -U trails -d postgres -tAc \ + "SELECT 1 FROM pg_database WHERE datname='$DB'" \ + | grep -q 1 \ + || docker compose exec -T postgres createdb -U trails "$DB" + docker compose exec -T postgres psql -U trails -d "$DB" -c \ + "CREATE EXTENSION IF NOT EXISTS postgis" + + # Pull, migrate, deploy (journal-only — no --profile means planner skipped) + docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file staging.env pull journal + docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file staging.env run --rm journal npx drizzle-kit push --config /app/packages/db/drizzle.config.ts --force + docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file staging.env up -d --remove-orphans journal + + # Reload Caddy to pick up the per-PR snippet (writes/replaces it from the SCP step) + docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile + + docker compose -f docker-compose.staging.yml -p "$PROJECT" ps + + - name: Comment preview URL on PR + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.number }} + body: | + 🚀 **PR preview deployed** + + - **Journal:** https://${{ steps.ports.outputs.host }} + - **Planner (shared staging):** https://planner.staging.trails.cool + - **Database:** `${{ steps.ports.outputs.database }}` (separate from production / persistent staging) + + Updates automatically on push. Tears down when this PR closes. + + # ── PR preview teardown ────────────────────────────────────────────── + teardown-preview: + name: Tear Down PR Preview + if: github.event_name == 'pull_request' && github.event.action == 'closed' + runs-on: ubuntu-latest + environment: production + permissions: + pull-requests: write + steps: + - id: ports + name: Compute project + database name + run: | + PR=${{ github.event.number }} + echo "pr=$PR" >> "$GITHUB_OUTPUT" + echo "project=trails-pr-$PR" >> "$GITHUB_OUTPUT" + echo "database=trails_pr_$PR" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@v6 + + - name: Decrypt secrets (needed to satisfy compose env vars during down) + run: | + curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64 + chmod +x sops-v3.9.4.linux.amd64 + SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > infrastructure/staging.env + { + echo "DOMAIN=pr-${{ steps.ports.outputs.pr }}.staging.trails.cool" + echo "STAGING_DATABASE=${{ steps.ports.outputs.database }}" + echo "JOURNAL_HOST_PORT=$((3200 + 2 * ${{ steps.ports.outputs.pr }}))" + echo "PLANNER_HOST_PORT=$((3201 + 2 * ${{ steps.ports.outputs.pr }}))" + } >> infrastructure/staging.env + + - name: Copy compose + env (teardown still needs the file) + uses: appleboy/scp-action@v1 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: root + key: ${{ secrets.DEPLOY_SSH_KEY }} + source: "infrastructure/docker-compose.staging.yml,infrastructure/staging.env" + target: /opt/trails-cool + strip_components: 1 + + - name: Tear down via SSH + uses: appleboy/ssh-action@v1 + env: + PR: ${{ steps.ports.outputs.pr }} + PROJECT: ${{ steps.ports.outputs.project }} + DB: ${{ steps.ports.outputs.database }} + with: + host: ${{ secrets.DEPLOY_HOST }} + username: root + key: ${{ secrets.DEPLOY_SSH_KEY }} + envs: PR,PROJECT,DB + script: | + set -euo pipefail + cd /opt/trails-cool + + # Stop and remove containers + volumes for this PR + docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file staging.env down --remove-orphans || true + + # Drop the per-PR database (idempotent) + docker compose exec -T postgres dropdb -U trails --if-exists "$DB" || true + + # Remove the per-PR Caddy snippet and reload + rm -f "sites/pr-$PR.caddyfile" + docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile || true + + - name: Find existing preview comment + uses: peter-evans/find-comment@v3 + id: find-comment + with: + issue-number: ${{ github.event.number }} + comment-author: "github-actions[bot]" + body-includes: "PR preview deployed" + + - name: Update preview comment on close + if: steps.find-comment.outputs.comment-id + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + edit-mode: replace + body: | + 🧹 PR preview torn down (PR ${{ github.event.action == 'closed' && github.event.pull_request.merged && 'merged' || 'closed' }}). diff --git a/.github/workflows/staging-cleanup.yml b/.github/workflows/staging-cleanup.yml new file mode 100644 index 0000000..595888f --- /dev/null +++ b/.github/workflows/staging-cleanup.yml @@ -0,0 +1,112 @@ +name: Staging Cleanup + +# Sweeps the production server for orphaned PR-preview resources whose PRs +# have closed without the cd-staging teardown job running (e.g., the +# teardown failed, the workflow file was changed mid-flight, or the PR was +# closed while runners were down). Runs weekly and can be triggered ad-hoc. + +on: + schedule: + # Every Monday at 04:00 UTC + - cron: "0 4 * * 1" + workflow_dispatch: {} + +concurrency: + group: staging-cleanup + cancel-in-progress: false + +jobs: + cleanup: + name: Sweep orphaned previews + runs-on: ubuntu-latest + environment: production + permissions: + contents: read + pull-requests: read + steps: + - name: List active preview projects + id: list + uses: appleboy/ssh-action@v1 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: root + key: ${{ secrets.DEPLOY_SSH_KEY }} + script_stop: true + script: | + cd /opt/trails-cool + # Emit one project name per line, e.g. "trails-pr-123" + docker compose ls --format json --filter "name=trails-pr-" \ + | python3 -c 'import json,sys + try: + data = json.load(sys.stdin) + except Exception: + data = [] + for d in data: + n = d.get("Name","") + if n.startswith("trails-pr-"): + print(n)' \ + || true + + - name: Determine which PRs are still open + id: orphans + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PROJECTS: ${{ steps.list.outputs.stdout }} + run: | + set -euo pipefail + ORPHANS=() + if [ -z "${PROJECTS:-}" ]; then + echo "No active preview projects." + echo "orphans=" >> "$GITHUB_OUTPUT" + exit 0 + fi + while IFS= read -r project; do + [ -z "$project" ] && continue + pr="${project#trails-pr-}" + # If gh can't find the PR (deleted) or it's not OPEN, treat as orphan. + state=$(gh pr view "$pr" --repo "${{ github.repository }}" --json state -q .state 2>/dev/null || echo "MISSING") + if [ "$state" != "OPEN" ]; then + echo "Orphan: PR #$pr (state=$state) → tear down $project" + ORPHANS+=("$pr") + fi + done <<< "${PROJECTS}" + IFS=, + echo "orphans=${ORPHANS[*]:-}" >> "$GITHUB_OUTPUT" + + - name: Tear down orphans + if: steps.orphans.outputs.orphans != '' + env: + ORPHANS: ${{ steps.orphans.outputs.orphans }} + uses: appleboy/ssh-action@v1 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: root + key: ${{ secrets.DEPLOY_SSH_KEY }} + envs: ORPHANS + script: | + set -euo pipefail + cd /opt/trails-cool + IFS=, read -ra PRS <<< "$ORPHANS" + for PR in "${PRS[@]}"; do + [ -z "$PR" ] && continue + PROJECT="trails-pr-$PR" + DB="trails_pr_$PR" + echo "→ tearing down $PROJECT" + # `down` needs the same env file the deploy used; staging.env on + # disk may belong to a different PR, so synthesize a minimal one. + cat > /tmp/cleanup.env <.staging.trails.cool` | `trails_pr_` | PR open/sync | + +PR previews are **journal-only** — their `PLANNER_URL` points at the persistent staging planner so we don't pay 256MB per preview for an extra planner. The persistent staging planner's CSP allows `connect-src wss://*.staging.trails.cool` so PR-preview journals can talk to it. + +**Port scheme** (host's loopback, reverse-proxied by Caddy via `host.docker.internal`): +- Persistent staging: journal `3100`, planner `3101` +- PR `` preview: journal `3200 + 2N`, planner `3201 + 2N` (planner unused for previews) + +**Compose project namespacing** keeps each preview isolated: +- Persistent staging: `-p trails-staging` +- PR ``: `-p trails-pr-` + +The shared file `infrastructure/docker-compose.staging.yml` covers both — env vars (`DOMAIN`, `STAGING_DATABASE`, `JOURNAL_HOST_PORT`, `JOURNAL_IMAGE_TAG`, …) parametrize per target. Persistent staging uses `--profile persistent` to also start the planner; PR previews omit the profile. + +**Caddy routing.** Persistent staging has fixed site blocks in `infrastructure/Caddyfile`. Per-PR site blocks are written by `cd-staging.yml` to `/opt/trails-cool/sites/pr-.caddyfile` (mounted into Caddy at `/etc/caddy/sites/`) and picked up via `import sites/*.caddyfile` on a Caddy reload. No on-demand TLS; standard automatic HTTPS issues a per-host cert. + +**Database isolation.** Each preview gets its own database on the production Postgres instance, schema applied via `drizzle-kit push --force`. Created on PR open, dropped on close. The persistent staging DB is never touched by previews. + +**Concurrent preview cap.** Max 3 concurrent PR previews. When a 4th opens, the deploy job evicts the oldest project before deploying. + +**Cleanup.** `cd-staging.yml`'s teardown job runs on PR close. `staging-cleanup.yml` runs weekly to catch orphans whose teardown never ran. + +**Debugging.** SSH to the flagship (`ssh -i ~/.ssh/trails-cool-deploy root@trails.cool`) and run `docker compose -f docker-compose.staging.yml -p trails-pr- logs -f` to tail a preview. `docker compose ls --filter name=trails-pr-` shows everything currently up. + ## OpenSpec Workflow Specs live in `openspec/`. Use these slash commands: diff --git a/infrastructure/Caddyfile b/infrastructure/Caddyfile index 6d80f0a..11bc024 100644 --- a/infrastructure/Caddyfile +++ b/infrastructure/Caddyfile @@ -69,3 +69,52 @@ planner.{$DOMAIN:trails.cool} { lb_try_interval 250ms } } + +# ── Staging ────────────────────────────────────────────────────────────── +# Persistent staging instance. CSP allow-lists hardcode `staging.trails.cool` +# rather than `{$DOMAIN}` because the Caddy container runs with the +# production DOMAIN env (`trails.cool`); staging blocks need their own domain +# baked in. Upstreams are reached via `host.docker.internal` because the +# staging compose project publishes its containers on the host's loopback +# (127.0.0.1:3100/3101) rather than joining the production Caddy network. + +staging.{$DOMAIN:trails.cool} { + import security_headers + import block_scanners + log { + output stdout + format json + } + reverse_proxy host.docker.internal:3100 { + lb_try_duration 30s + lb_try_interval 250ms + } +} + +planner.staging.{$DOMAIN:trails.cool} { + header { + Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" + X-Content-Type-Options "nosniff" + X-Frame-Options "DENY" + Referrer-Policy "strict-origin-when-cross-origin" + Permissions-Policy "camera=(), microphone=(), geolocation=()" + # connect-src includes wss + https://*.staging so PR-preview journals + # (pr-N.staging.trails.cool) can use this shared planner. PR previews + # are journal-only; this is the planner they all talk to. + Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' blob:; worker-src 'self' blob:; style-src 'self' 'unsafe-inline'; img-src 'self' data: https://*.tile.openstreetmap.org; connect-src 'self' wss://*.staging.{$DOMAIN:trails.cool} https://*.staging.{$DOMAIN:trails.cool} https://staging.{$DOMAIN:trails.cool} https://*.sentry.io https://*.ingest.de.sentry.io; font-src 'self';" + } + import block_scanners + log { + output stdout + format json + } + reverse_proxy host.docker.internal:3101 { + lb_try_duration 30s + lb_try_interval 250ms + } +} + +# Per-PR preview snippets are written by cd-staging.yml into +# /etc/caddy/sites/pr-.caddyfile and picked up here on Caddy reload. The +# glob is allowed to match nothing — Caddy treats an empty match as a no-op. +import /etc/caddy/sites/*.caddyfile diff --git a/infrastructure/docker-compose.staging.yml b/infrastructure/docker-compose.staging.yml new file mode 100644 index 0000000..1f91537 --- /dev/null +++ b/infrastructure/docker-compose.staging.yml @@ -0,0 +1,100 @@ +# Staging / PR-preview compose file. +# +# Used for both the persistent staging stack and ephemeral PR previews. +# The cd-staging.yml workflow picks the project name and fills in the host +# ports + DOMAIN + STAGING_DATABASE from the PR number (or "staging" for the +# persistent stack): +# +# # Persistent staging +# PROJECT=trails-staging +# JOURNAL_HOST_PORT=3100 PLANNER_HOST_PORT=3101 \ +# DOMAIN=staging.trails.cool STAGING_DATABASE=trails_staging \ +# docker compose -f docker-compose.staging.yml -p $PROJECT up -d +# +# # PR 123 preview (port = 3200 + 2N for journal, 3201 + 2N for planner) +# PROJECT=trails-pr-123 +# JOURNAL_HOST_PORT=3446 PLANNER_HOST_PORT=3447 \ +# DOMAIN=pr-123.staging.trails.cool STAGING_DATABASE=trails_pr_123 \ +# JOURNAL_IMAGE_TAG=pr-123 PLANNER_IMAGE_TAG=pr-123 \ +# docker compose -f docker-compose.staging.yml -p $PROJECT up -d +# +# `trails-shared` is created by the production compose file (see +# docker-compose.yml) so staging/preview containers can reach the production +# `postgres` host. BRouter lives on a separate host and is reached over +# vSwitch using the production-shared BROUTER_URL / BROUTER_AUTH_TOKEN. +# +# Container names are not pinned — Docker Compose generates them from the +# project name (e.g. `trails-pr-123-journal-1`), which keeps each preview +# isolated without manual naming. + +services: + journal: + image: ghcr.io/trails-cool/journal:${JOURNAL_IMAGE_TAG:-latest} + restart: unless-stopped + ports: + - "127.0.0.1:${JOURNAL_HOST_PORT:?JOURNAL_HOST_PORT must be set}:3000" + networks: + - trails-shared + environment: + DOMAIN: ${DOMAIN:?DOMAIN must be set} + ORIGIN: https://${DOMAIN} + # PR previews override this to point at the persistent staging + # planner (planner.staging.trails.cool). Persistent staging defaults + # to its own planner subdomain. + PLANNER_URL: ${PLANNER_URL:-https://planner.${DOMAIN}} + IS_FLAGSHIP: "" + DATABASE_URL: postgres://trails:${POSTGRES_PASSWORD:-trails}@postgres:5432/${STAGING_DATABASE:?STAGING_DATABASE must be set} + JWT_SECRET: ${JWT_SECRET:?JWT_SECRET must be set} + SESSION_SECRET: ${SESSION_SECRET:?SESSION_SECRET must be set} + NODE_ENV: production + PORT: 3000 + SENTRY_RELEASE: ${SENTRY_RELEASE:-} + SMTP_URL: "" + SMTP_FROM: trails.cool staging + WAHOO_CLIENT_ID: "" + WAHOO_CLIENT_SECRET: "" + WAHOO_WEBHOOK_TOKEN: "" + DEMO_BOT_ENABLED: "" + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:3000/api/health || exit 1"] + interval: 15s + timeout: 5s + retries: 3 + deploy: + resources: + limits: + memory: 256M + + planner: + image: ghcr.io/trails-cool/planner:${PLANNER_IMAGE_TAG:-latest} + # Only the persistent staging stack runs a planner. PR previews are + # journal-only and point their PLANNER_URL at the persistent + # planner.staging.trails.cool. Saves ~256MB per active preview. + profiles: ["persistent"] + restart: unless-stopped + ports: + - "127.0.0.1:${PLANNER_HOST_PORT:-3101}:3001" + networks: + - trails-shared + environment: + BROUTER_URL: ${BROUTER_URL:?BROUTER_URL must be set} + BROUTER_AUTH_TOKEN: ${BROUTER_AUTH_TOKEN:?BROUTER_AUTH_TOKEN must be set} + OVERPASS_URLS: ${OVERPASS_URLS:-https://lz4.overpass-api.de/api/interpreter,https://overpass-api.de/api/interpreter} + DATABASE_URL: postgres://trails:${POSTGRES_PASSWORD:-trails}@postgres:5432/${STAGING_DATABASE} + NODE_ENV: production + PORT: 3001 + SENTRY_RELEASE: ${SENTRY_RELEASE:-} + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:3001/health || exit 1"] + interval: 15s + timeout: 5s + retries: 3 + deploy: + resources: + limits: + memory: 256M + +networks: + trails-shared: + external: true + name: trails-shared diff --git a/infrastructure/docker-compose.yml b/infrastructure/docker-compose.yml index ff7c592..4614663 100644 --- a/infrastructure/docker-compose.yml +++ b/infrastructure/docker-compose.yml @@ -6,10 +6,19 @@ services: - "80:80" - "443:443" - "443:443/udp" + # host.docker.internal lets Caddy reverse-proxy to staging / PR-preview + # containers, which publish on the host's loopback (e.g. 127.0.0.1:3100). + # Linux requires `host-gateway` to make this name resolve. + extra_hosts: + - "host.docker.internal:host-gateway" environment: DOMAIN: ${DOMAIN:-trails.cool} volumes: - ./Caddyfile:/etc/caddy/Caddyfile:ro + # Per-PR staging snippets dropped in by cd-staging.yml. Bind mount + # auto-creates the host directory if it doesn't exist, so a fresh + # server doesn't need pre-provisioning. + - ./sites:/etc/caddy/sites:ro - caddy_data:/data - caddy_config:/config depends_on: @@ -90,6 +99,12 @@ services: postgres: image: postgis/postgis:16-3.4 restart: unless-stopped + # Joined to the shared network so the staging compose project can reach + # this instance by hostname `postgres`. Production services keep using + # the default network as before. + networks: + - default + - trails-shared environment: POSTGRES_USER: trails POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-trails} @@ -226,6 +241,16 @@ services: # - garage_data:/var/lib/garage # - ./garage.toml:/etc/garage.toml:ro +networks: + # Default project network — kept implicit for production services. + default: + # Created by this compose project with a fixed (un-namespaced) name so the + # staging compose project can attach to it via `external: true`. Only + # postgres is joined here on the production side; staging containers join + # to reach postgres by hostname. + trails-shared: + name: trails-shared + volumes: caddy_data: caddy_config: diff --git a/infrastructure/staging.env.template b/infrastructure/staging.env.template new file mode 100644 index 0000000..b859098 --- /dev/null +++ b/infrastructure/staging.env.template @@ -0,0 +1,29 @@ +# Staging environment variables. The cd-staging.yml workflow assembles +# these from SOPS secrets + workflow-injected values and ships them to the +# server as `infrastructure/staging.env`. Keep this file in sync with the +# `environment:` blocks in `docker-compose.staging.yml`. +# +# For PR previews the same template is reused, with DOMAIN, STAGING_DATABASE, +# and image tags overridden per-PR by the workflow. + +# Domain the staging stack serves at. Persistent staging uses +# staging.trails.cool; PR previews use pr-.staging.trails.cool. +DOMAIN=staging.trails.cool + +# Database name on the shared production Postgres. Persistent staging uses +# trails_staging; PR previews use trails_pr_. +STAGING_DATABASE=trails_staging + +# Inherited from SOPS secrets.app.env (decrypted in the workflow): +POSTGRES_PASSWORD= +JWT_SECRET= +SESSION_SECRET= +BROUTER_URL= +BROUTER_AUTH_TOKEN= + +# Image tag to deploy. `latest` for staging, `pr-` for PR previews. +JOURNAL_IMAGE_TAG=latest +PLANNER_IMAGE_TAG=latest + +# Sentry release SHA, set by the workflow. +SENTRY_RELEASE= diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf index f2f8a69..3a23c84 100644 --- a/infrastructure/terraform/main.tf +++ b/infrastructure/terraform/main.tf @@ -191,6 +191,42 @@ resource "hcloud_zone_rrset" "planner_aaaa" { records = [{ value = hcloud_server.trails.ipv6_address }] } +# Staging — persistent journal at staging.trails.cool, persistent planner +# at planner.staging.trails.cool, and PR previews at pr-.staging.trails.cool +# (covered by the wildcard). All resolve to the flagship; Caddy routes per-host. + +resource "hcloud_zone_rrset" "staging_a" { + zone = "trails.cool" + name = "staging" + type = "A" + ttl = 300 + records = [{ value = hcloud_server.trails.ipv4_address }] +} + +resource "hcloud_zone_rrset" "staging_aaaa" { + zone = "trails.cool" + name = "staging" + type = "AAAA" + ttl = 300 + records = [{ value = hcloud_server.trails.ipv6_address }] +} + +resource "hcloud_zone_rrset" "staging_wildcard_a" { + zone = "trails.cool" + name = "*.staging" + type = "A" + ttl = 300 + records = [{ value = hcloud_server.trails.ipv4_address }] +} + +resource "hcloud_zone_rrset" "staging_wildcard_aaaa" { + zone = "trails.cool" + name = "*.staging" + type = "AAAA" + ttl = 300 + records = [{ value = hcloud_server.trails.ipv6_address }] +} + # Internal wildcard — all *.internal.trails.cool resolves to the server. # Caddy handles routing per-subdomain. No individual DNS entries needed # for internal services (Grafana, Prometheus, etc.) diff --git a/openspec/changes/staging-environments/design.md b/openspec/changes/staging-environments/design.md index 23b1a5e..6f615f6 100644 --- a/openspec/changes/staging-environments/design.md +++ b/openspec/changes/staging-environments/design.md @@ -27,10 +27,10 @@ The server has headroom for lightweight additional containers. Caddy handles aut **Rationale**: A separate Compose project gives clean namespace isolation (container names, volumes) without a second server. Sharing BRouter and monitoring avoids duplicating heavy services. **Alternative considered**: Docker Compose profiles — simpler but risks accidental cross-contamination between production and staging in the same project. -### 2. Caddy on-demand TLS with wildcard routing -**Choice**: Use Caddy's `on_demand_tls` with a wildcard site block for `*.staging.trails.cool`. A small validation endpoint confirms which subdomains are active before Caddy obtains a certificate. -**Rationale**: Avoids pre-configuring Caddy for each PR. Caddy automatically provisions TLS certificates on first request. The validation endpoint prevents abuse (random subdomains triggering cert issuance). -**Alternative considered**: Wildcard certificate via DNS challenge — requires DNS API credentials and more complex setup. +### 2. Per-PR Caddyfile snippets with reload +**Choice**: The main Caddyfile uses `import sites/*.caddyfile`. The cd-staging workflow writes a snippet per PR (e.g. `sites/pr-123.caddyfile`) on PR open and removes it on PR close, then reloads Caddy in-place. Standard automatic HTTPS issues a per-host cert. +**Rationale**: Caddy can't compute upstream ports from a regex capture (the spec's `3200 + 2N` formula), so a wildcard block would need a sidecar router service. A graceful Caddy reload is fast and idempotent, and the existing cd-apps workflow already reloads Caddy on every deploy — extending that pattern is simpler than adding a new service. +**Alternatives considered**: (a) Wildcard `*.staging.trails.cool` block with on-demand TLS plus a small Node router that handles Caddy's `ask` check and wildcard host→port proxying — more moving parts. (b) DNS-challenge wildcard certificate — requires DNS API credentials in Caddy and more complex setup. ### 3. Per-PR databases in shared PostgreSQL **Choice**: PR previews use per-PR databases (`trails_pr_123`) in the production PostgreSQL instance. Staging uses `trails_staging`. Created by the workflow, dropped on PR close. diff --git a/openspec/changes/staging-environments/specs/infrastructure/spec.md b/openspec/changes/staging-environments/specs/infrastructure/spec.md index b670ca0..9667709 100644 --- a/openspec/changes/staging-environments/specs/infrastructure/spec.md +++ b/openspec/changes/staging-environments/specs/infrastructure/spec.md @@ -1,7 +1,7 @@ ## MODIFIED Requirements ### Requirement: Caddy reverse proxy routing -Caddy SHALL route requests to staging and PR preview containers via wildcard subdomain matching, in addition to the existing production routing. +Caddy SHALL route requests to staging and PR preview containers via per-host site blocks, in addition to the existing production routing. #### Scenario: Staging subdomain routing - **WHEN** a request arrives for `staging.trails.cool` @@ -13,12 +13,14 @@ Caddy SHALL route requests to staging and PR preview containers via wildcard sub #### Scenario: PR preview routing - **WHEN** a request arrives for `pr-123.staging.trails.cool` -- **THEN** Caddy proxies it to the PR 123 journal container on the correct dynamically assigned port +- **THEN** Caddy proxies it to the PR 123 journal container on its assigned port (`3200 + 2N`) -#### Scenario: On-demand TLS for staging subdomains -- **WHEN** a first request arrives for a new staging subdomain -- **THEN** Caddy automatically provisions a TLS certificate via Let's Encrypt -- **AND** a validation endpoint confirms the subdomain is an active staging/preview environment before certificate issuance +#### Scenario: Per-PR Caddyfile snippet lifecycle +- **WHEN** a PR preview is deployed +- **THEN** the cd-staging workflow writes a Caddyfile snippet at `sites/pr-.caddyfile` and reloads Caddy +- **WHEN** a PR is closed +- **THEN** the workflow removes the snippet and reloads Caddy +- **AND** standard automatic HTTPS issues / retains the per-host certificate via Let's Encrypt ### Requirement: Docker Compose deployment The staging environment SHALL be deployed as a separate Docker Compose project alongside production on the same server. diff --git a/openspec/changes/staging-environments/tasks.md b/openspec/changes/staging-environments/tasks.md index 9c0d1cd..e46ade3 100644 --- a/openspec/changes/staging-environments/tasks.md +++ b/openspec/changes/staging-environments/tasks.md @@ -1,38 +1,38 @@ ## 1. DNS & TLS Setup -- [ ] 1.1 Add wildcard DNS record `*.staging.trails.cool` pointing to the Hetzner server IP -- [ ] 1.2 Add `staging.trails.cool` and `planner.staging.trails.cool` DNS A records +- [x] 1.1 Add wildcard DNS record `*.staging.trails.cool` pointing to the Hetzner server IP +- [x] 1.2 Add `staging.trails.cool` and `planner.staging.trails.cool` DNS A records (planner.staging covered by the wildcard) ## 2. Docker Compose Staging Configuration -- [ ] 2.1 Create `infrastructure/docker-compose.staging.yml` with staging journal (port 3100), planner (port 3101), memory limits (256MB), and `trails_staging` database URL -- [ ] 2.2 Create `infrastructure/staging.env.template` documenting required staging environment variables (DOMAIN, DATABASE_URL, JWT_SECRET, SESSION_SECRET) -- [ ] 2.3 Add a shared Docker network (`trails-shared`) to production `docker-compose.yml` so staging can reach BRouter and PostgreSQL +- [x] 2.1 Create `infrastructure/docker-compose.staging.yml` with staging journal (port 3100), planner (port 3101), memory limits (256MB), and `trails_staging` database URL +- [x] 2.2 Create `infrastructure/staging.env.template` documenting required staging environment variables (DOMAIN, DATABASE_URL, JWT_SECRET, SESSION_SECRET) +- [x] 2.3 Add a shared Docker network (`trails-shared`) to production `docker-compose.yml` so staging can reach BRouter and PostgreSQL - [ ] 2.4 Verify staging containers start with `docker compose -f docker-compose.staging.yml -p trails-staging up -d` on the server ## 3. Caddy Wildcard Routing -- [ ] 3.1 Add `staging.trails.cool` site block proxying to journal on port 3100 -- [ ] 3.2 Add `planner.staging.trails.cool` site block proxying to planner on port 3101 -- [ ] 3.3 Add `*.staging.trails.cool` wildcard site block with on-demand TLS for PR previews — extract PR number from subdomain, proxy to `localhost:3200 + (PR * 2)` -- [ ] 3.4 Create a TLS validation endpoint (small script or Caddy matcher) that checks if the requested subdomain corresponds to a running container +- [x] 3.1 Add `staging.trails.cool` site block proxying to journal on port 3100 +- [x] 3.2 Add `planner.staging.trails.cool` site block proxying to planner on port 3101 +- [x] 3.3 Add `import sites/*.caddyfile` to the main Caddyfile so per-PR site blocks can be dropped in and picked up on reload +- [x] 3.4 Define the per-PR Caddyfile snippet template the cd-staging workflow writes for each preview (host = `pr-.staging.trails.cool`, upstream = `host.docker.internal:`, port = `3200 + 2N`) - [ ] 3.5 Reload Caddy and verify staging routes work with `curl -sf https://staging.trails.cool/api/health` ## 4. GitHub Actions Workflow -- [ ] 4.1 Create `.github/workflows/cd-staging.yml` triggered on push to main (paths: `apps/`, `packages/`) and on PR open/synchronize/close (same paths) -- [ ] 4.2 Implement the **staging deploy** job: build images, SSH to server, `docker compose -f docker-compose.staging.yml -p trails-staging pull && up -d`, run Drizzle push against `trails_staging` -- [ ] 4.3 Implement the **PR preview deploy** job: compute ports from PR number, create `trails_pr_` database if not exists, build images tagged with PR number, deploy containers, post preview URL as PR comment -- [ ] 4.4 Implement the **PR preview teardown** job: stop and remove PR containers, drop `trails_pr_` database, delete PR comment -- [ ] 4.5 Add the concurrent preview limit check: if >3 active previews, tear down the oldest before deploying a new one +- [x] 4.1 Create `.github/workflows/cd-staging.yml` triggered on push to main (paths: `apps/`, `packages/`) and on PR open/synchronize/close (same paths) +- [x] 4.2 Implement the **staging deploy** job: build images, SSH to server, `docker compose -f docker-compose.staging.yml -p trails-staging pull && up -d`, run Drizzle push against `trails_staging` +- [x] 4.3 Implement the **PR preview deploy** job: compute ports from PR number, create `trails_pr_` database if not exists, build images tagged with PR number, deploy containers, post preview URL as PR comment +- [x] 4.4 Implement the **PR preview teardown** job: stop and remove PR containers, drop `trails_pr_` database, delete PR comment +- [x] 4.5 Add the concurrent preview limit check: if >3 active previews, tear down the oldest before deploying a new one ## 5. Cleanup & Safety -- [ ] 5.1 Create a scheduled cleanup job (weekly cron in GitHub Actions or pg-boss on the server) that lists running `trails-pr-*` containers, checks PR status via `gh pr view`, and tears down orphans -- [ ] 5.2 Add memory limits (`deploy.resources.limits.memory: 256m`) to staging containers in the compose override +- [x] 5.1 Create a scheduled cleanup job (weekly cron in GitHub Actions or pg-boss on the server) that lists running `trails-pr-*` containers, checks PR status via `gh pr view`, and tears down orphans +- [x] 5.2 Add memory limits (`deploy.resources.limits.memory: 256m`) to staging containers in the compose override - [ ] 5.3 Test full lifecycle: open a test PR → verify preview deploys → push a commit → verify preview updates → close PR → verify teardown ## 6. Documentation -- [ ] 6.1 Add a "Staging & Previews" section to CLAUDE.md documenting the staging URL, PR preview URL pattern, port scheme, and how to debug staging issues -- [ ] 6.2 Update the Deployment table in CLAUDE.md with the new `cd-staging.yml` workflow +- [x] 6.1 Add a "Staging & Previews" section to CLAUDE.md documenting the staging URL, PR preview URL pattern, port scheme, and how to debug staging issues +- [x] 6.2 Update the Deployment table in CLAUDE.md with the new `cd-staging.yml` workflow