Merge pull request #356 from trails-cool/stigi/staging-envs

Add staging + PR-preview environments on the flagship
This commit is contained in:
Ullrich Schäfer 2026-05-03 22:13:55 +02:00 committed by GitHub
commit de5c96351c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 818 additions and 29 deletions

402
.github/workflows/cd-staging.yml vendored Normal file
View file

@ -0,0 +1,402 @@
name: CD Staging
# Builds, deploys, and tears down the persistent staging stack and per-PR
# preview environments. See `openspec/changes/staging-environments/` for the
# design (decisions on shared Postgres, port allocation, per-PR Caddyfile
# snippets) and CLAUDE.md "Staging & Previews" for the operator-facing view.
on:
push:
branches: [main]
paths:
- "apps/**"
- "packages/**"
- "pnpm-lock.yaml"
pull_request:
types: [opened, synchronize, reopened, closed]
paths:
- "apps/**"
- "packages/**"
- "pnpm-lock.yaml"
workflow_dispatch: {}
# Per-target concurrency: persistent staging deploys serialize against
# themselves, each PR's preview lifecycle serializes against itself.
concurrency:
group: staging-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.number) || 'main' }}
cancel-in-progress: true
jobs:
# ── Build ─────────────────────────────────────────────────────────────
# Tags:
# main push → :staging + :<sha>
# PR open/sync/reopen → :pr-<N> + :pr-<N>-<sha>
# Skipped entirely on PR close (teardown doesn't need new images).
build-images:
name: Build & Push Docker Images
if: github.event_name != 'pull_request' || github.event.action != 'closed'
runs-on: ubuntu-latest
environment: production
permissions:
contents: read
packages: write
strategy:
matrix:
app: [journal, planner]
outputs:
tag_primary: ${{ steps.tags.outputs.primary }}
tag_sha: ${{ steps.tags.outputs.sha }}
steps:
- uses: actions/checkout@v6
- id: tags
name: Compute image tags
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
PRIMARY="pr-${{ github.event.number }}"
SHA="pr-${{ github.event.number }}-${{ github.event.pull_request.head.sha }}"
else
PRIMARY="staging"
SHA="${{ github.sha }}"
fi
echo "primary=$PRIMARY" >> "$GITHUB_OUTPUT"
echo "sha=$SHA" >> "$GITHUB_OUTPUT"
- uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Decrypt Sentry auth token
run: |
curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64
chmod +x sops-v3.9.4.linux.amd64
SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > /tmp/secrets.env
grep SENTRY_AUTH_TOKEN /tmp/secrets.env | cut -d= -f2- | tr -d '\n' > /tmp/sentry_token
- uses: docker/build-push-action@v7
with:
context: .
file: apps/${{ matrix.app }}/Dockerfile
push: true
tags: |
ghcr.io/trails-cool/${{ matrix.app }}:${{ steps.tags.outputs.primary }}
ghcr.io/trails-cool/${{ matrix.app }}:${{ steps.tags.outputs.sha }}
build-args: |
SENTRY_RELEASE=${{ steps.tags.outputs.sha }}
secrets: |
SENTRY_AUTH_TOKEN=/tmp/sentry_token
# ── Deploy persistent staging (main push) ────────────────────────────
deploy-staging:
name: Deploy Staging
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
needs: [build-images]
runs-on: ubuntu-latest
environment: production
steps:
- uses: actions/checkout@v6
- name: Decrypt secrets
run: |
curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64
chmod +x sops-v3.9.4.linux.amd64
SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > infrastructure/staging.env
{
echo "DOMAIN=staging.trails.cool"
echo "STAGING_DATABASE=trails_staging"
echo "JOURNAL_HOST_PORT=3100"
echo "PLANNER_HOST_PORT=3101"
echo "JOURNAL_IMAGE_TAG=staging"
echo "PLANNER_IMAGE_TAG=staging"
echo "SENTRY_RELEASE=${{ github.sha }}"
} >> infrastructure/staging.env
- name: Copy compose file + env to server
uses: appleboy/scp-action@v1
with:
host: ${{ secrets.DEPLOY_HOST }}
username: root
key: ${{ secrets.DEPLOY_SSH_KEY }}
source: "infrastructure/docker-compose.staging.yml,infrastructure/staging.env"
target: /opt/trails-cool
strip_components: 1
- name: Deploy via SSH
uses: appleboy/ssh-action@v1
with:
host: ${{ secrets.DEPLOY_HOST }}
username: root
key: ${{ secrets.DEPLOY_SSH_KEY }}
script: |
set -euo pipefail
cd /opt/trails-cool
GHCR_TOKEN=$(grep DEPLOY_GHCR_TOKEN staging.env | cut -d= -f2-)
echo "$GHCR_TOKEN" | docker login ghcr.io -u stigi --password-stdin
# Bootstrap the shared network so cd-staging works regardless of
# whether cd-infra has already run with the new docker-compose.yml.
# Once cd-infra runs, postgres is permanently joined via compose;
# until then, attach it imperatively.
docker network inspect trails-shared >/dev/null 2>&1 || docker network create trails-shared
PG_CONTAINER=$(docker ps --filter "name=trails-cool-postgres" --format '{{.Names}}' | head -1)
if [ -n "$PG_CONTAINER" ]; then
docker network connect trails-shared "$PG_CONTAINER" 2>/dev/null || true
fi
# Ensure trails_staging database exists with postgis. Production
# init scripts only run on first data-dir init, so a freshly
# created database has no extensions.
docker compose exec -T postgres psql -U trails -d postgres -tAc \
"SELECT 1 FROM pg_database WHERE datname='trails_staging'" \
| grep -q 1 \
|| docker compose exec -T postgres createdb -U trails trails_staging
docker compose exec -T postgres psql -U trails -d trails_staging -c \
"CREATE EXTENSION IF NOT EXISTS postgis"
# Pull and deploy staging containers (journal + planner via "persistent" profile)
docker compose -f docker-compose.staging.yml -p trails-staging --env-file staging.env --profile persistent pull
docker compose -f docker-compose.staging.yml -p trails-staging --env-file staging.env --profile persistent run --rm journal npx drizzle-kit push --config /app/packages/db/drizzle.config.ts --force
docker compose -f docker-compose.staging.yml -p trails-staging --env-file staging.env --profile persistent up -d --remove-orphans
# Reload Caddy so new staging routes (or Caddyfile changes shipped
# via cd-infra) are live. Idempotent.
docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile || true
docker compose -f docker-compose.staging.yml -p trails-staging ps
# ── PR preview deploy ────────────────────────────────────────────────
deploy-preview:
name: Deploy PR Preview
if: github.event_name == 'pull_request' && github.event.action != 'closed'
needs: [build-images]
runs-on: ubuntu-latest
environment: production
permissions:
pull-requests: write
steps:
- uses: actions/checkout@v6
- id: ports
name: Compute preview ports + project name
run: |
PR=${{ github.event.number }}
# journal = 3200 + 2N, planner unused (PR previews are journal-only)
JOURNAL_PORT=$((3200 + 2 * PR))
PLANNER_PORT=$((3201 + 2 * PR))
echo "pr=$PR" >> "$GITHUB_OUTPUT"
echo "journal_port=$JOURNAL_PORT" >> "$GITHUB_OUTPUT"
echo "planner_port=$PLANNER_PORT" >> "$GITHUB_OUTPUT"
echo "host=pr-$PR.staging.trails.cool" >> "$GITHUB_OUTPUT"
echo "project=trails-pr-$PR" >> "$GITHUB_OUTPUT"
echo "database=trails_pr_$PR" >> "$GITHUB_OUTPUT"
- name: Decrypt secrets + assemble env
run: |
curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64
chmod +x sops-v3.9.4.linux.amd64
SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > infrastructure/staging.env
{
echo "DOMAIN=${{ steps.ports.outputs.host }}"
echo "STAGING_DATABASE=${{ steps.ports.outputs.database }}"
echo "JOURNAL_HOST_PORT=${{ steps.ports.outputs.journal_port }}"
echo "PLANNER_HOST_PORT=${{ steps.ports.outputs.planner_port }}"
echo "JOURNAL_IMAGE_TAG=pr-${{ steps.ports.outputs.pr }}"
echo "PLANNER_IMAGE_TAG=pr-${{ steps.ports.outputs.pr }}"
# PR-preview journals all share the persistent staging planner.
echo "PLANNER_URL=https://planner.staging.trails.cool"
echo "SENTRY_RELEASE=${{ github.event.pull_request.head.sha }}"
} >> infrastructure/staging.env
- name: Generate per-PR Caddyfile snippet
run: |
mkdir -p infrastructure/sites
cat > infrastructure/sites/pr-${{ steps.ports.outputs.pr }}.caddyfile <<EOF
# Auto-generated by cd-staging.yml for PR ${{ steps.ports.outputs.pr }}. Do not edit.
${{ steps.ports.outputs.host }} {
import security_headers
import block_scanners
log {
output stdout
format json
}
reverse_proxy host.docker.internal:${{ steps.ports.outputs.journal_port }} {
lb_try_duration 30s
lb_try_interval 250ms
}
}
EOF
- name: Copy files to server
uses: appleboy/scp-action@v1
with:
host: ${{ secrets.DEPLOY_HOST }}
username: root
key: ${{ secrets.DEPLOY_SSH_KEY }}
source: "infrastructure/docker-compose.staging.yml,infrastructure/staging.env,infrastructure/sites/pr-${{ steps.ports.outputs.pr }}.caddyfile"
target: /opt/trails-cool
strip_components: 1
- name: Deploy preview via SSH
uses: appleboy/ssh-action@v1
env:
PR: ${{ steps.ports.outputs.pr }}
PROJECT: ${{ steps.ports.outputs.project }}
DB: ${{ steps.ports.outputs.database }}
with:
host: ${{ secrets.DEPLOY_HOST }}
username: root
key: ${{ secrets.DEPLOY_SSH_KEY }}
envs: PR,PROJECT,DB
script: |
set -euo pipefail
cd /opt/trails-cool
GHCR_TOKEN=$(grep DEPLOY_GHCR_TOKEN staging.env | cut -d= -f2-)
echo "$GHCR_TOKEN" | docker login ghcr.io -u stigi --password-stdin
# Same network bootstrap as deploy-staging — see comment there.
docker network inspect trails-shared >/dev/null 2>&1 || docker network create trails-shared
PG_CONTAINER=$(docker ps --filter "name=trails-cool-postgres" --format '{{.Names}}' | head -1)
if [ -n "$PG_CONTAINER" ]; then
docker network connect trails-shared "$PG_CONTAINER" 2>/dev/null || true
fi
# Concurrent preview limit (max 3): if we're at the cap and this
# PR isn't already running, evict the oldest preview project.
ACTIVE=$(docker compose ls --format json --filter "name=trails-pr-" | python3 -c 'import json,sys; data=json.load(sys.stdin); print("\n".join(d["Name"] for d in data))' 2>/dev/null || true)
if [ -n "$ACTIVE" ] && ! echo "$ACTIVE" | grep -qx "$PROJECT"; then
COUNT=$(echo "$ACTIVE" | grep -c '^trails-pr-' || true)
if [ "$COUNT" -ge 3 ]; then
# Pick the oldest by container CreatedAt of any service in the project.
OLDEST=$(docker ps -a --filter "name=trails-pr-" --format '{{.Names}} {{.CreatedAt}}' \
| awk '{ split($1,a,"-"); print "trails-pr-"a[3], $2" "$3" "$4 }' \
| sort -k2 \
| head -1 \
| awk '{print $1}')
if [ -n "$OLDEST" ] && [ "$OLDEST" != "$PROJECT" ]; then
echo "At cap; evicting oldest preview: $OLDEST"
OLD_PR=${OLDEST#trails-pr-}
docker compose -f docker-compose.staging.yml -p "$OLDEST" --env-file staging.env down --remove-orphans || true
docker compose exec -T postgres dropdb -U trails --if-exists "trails_pr_$OLD_PR" || true
rm -f "sites/pr-$OLD_PR.caddyfile"
fi
fi
fi
# Ensure per-PR database exists with postgis. (See deploy-staging
# for why we have to enable the extension explicitly.)
docker compose exec -T postgres psql -U trails -d postgres -tAc \
"SELECT 1 FROM pg_database WHERE datname='$DB'" \
| grep -q 1 \
|| docker compose exec -T postgres createdb -U trails "$DB"
docker compose exec -T postgres psql -U trails -d "$DB" -c \
"CREATE EXTENSION IF NOT EXISTS postgis"
# Pull, migrate, deploy (journal-only — no --profile means planner skipped)
docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file staging.env pull journal
docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file staging.env run --rm journal npx drizzle-kit push --config /app/packages/db/drizzle.config.ts --force
docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file staging.env up -d --remove-orphans journal
# Reload Caddy to pick up the per-PR snippet (writes/replaces it from the SCP step)
docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile
docker compose -f docker-compose.staging.yml -p "$PROJECT" ps
- name: Comment preview URL on PR
uses: peter-evans/create-or-update-comment@v4
with:
issue-number: ${{ github.event.number }}
body: |
🚀 **PR preview deployed**
- **Journal:** https://${{ steps.ports.outputs.host }}
- **Planner (shared staging):** https://planner.staging.trails.cool
- **Database:** `${{ steps.ports.outputs.database }}` (separate from production / persistent staging)
Updates automatically on push. Tears down when this PR closes.
# ── PR preview teardown ──────────────────────────────────────────────
teardown-preview:
name: Tear Down PR Preview
if: github.event_name == 'pull_request' && github.event.action == 'closed'
runs-on: ubuntu-latest
environment: production
permissions:
pull-requests: write
steps:
- id: ports
name: Compute project + database name
run: |
PR=${{ github.event.number }}
echo "pr=$PR" >> "$GITHUB_OUTPUT"
echo "project=trails-pr-$PR" >> "$GITHUB_OUTPUT"
echo "database=trails_pr_$PR" >> "$GITHUB_OUTPUT"
- uses: actions/checkout@v6
- name: Decrypt secrets (needed to satisfy compose env vars during down)
run: |
curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64
chmod +x sops-v3.9.4.linux.amd64
SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > infrastructure/staging.env
{
echo "DOMAIN=pr-${{ steps.ports.outputs.pr }}.staging.trails.cool"
echo "STAGING_DATABASE=${{ steps.ports.outputs.database }}"
echo "JOURNAL_HOST_PORT=$((3200 + 2 * ${{ steps.ports.outputs.pr }}))"
echo "PLANNER_HOST_PORT=$((3201 + 2 * ${{ steps.ports.outputs.pr }}))"
} >> infrastructure/staging.env
- name: Copy compose + env (teardown still needs the file)
uses: appleboy/scp-action@v1
with:
host: ${{ secrets.DEPLOY_HOST }}
username: root
key: ${{ secrets.DEPLOY_SSH_KEY }}
source: "infrastructure/docker-compose.staging.yml,infrastructure/staging.env"
target: /opt/trails-cool
strip_components: 1
- name: Tear down via SSH
uses: appleboy/ssh-action@v1
env:
PR: ${{ steps.ports.outputs.pr }}
PROJECT: ${{ steps.ports.outputs.project }}
DB: ${{ steps.ports.outputs.database }}
with:
host: ${{ secrets.DEPLOY_HOST }}
username: root
key: ${{ secrets.DEPLOY_SSH_KEY }}
envs: PR,PROJECT,DB
script: |
set -euo pipefail
cd /opt/trails-cool
# Stop and remove containers + volumes for this PR
docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file staging.env down --remove-orphans || true
# Drop the per-PR database (idempotent)
docker compose exec -T postgres dropdb -U trails --if-exists "$DB" || true
# Remove the per-PR Caddy snippet and reload
rm -f "sites/pr-$PR.caddyfile"
docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile || true
- name: Find existing preview comment
uses: peter-evans/find-comment@v3
id: find-comment
with:
issue-number: ${{ github.event.number }}
comment-author: "github-actions[bot]"
body-includes: "PR preview deployed"
- name: Update preview comment on close
if: steps.find-comment.outputs.comment-id
uses: peter-evans/create-or-update-comment@v4
with:
comment-id: ${{ steps.find-comment.outputs.comment-id }}
edit-mode: replace
body: |
🧹 PR preview torn down (PR ${{ github.event.action == 'closed' && github.event.pull_request.merged && 'merged' || 'closed' }}).

112
.github/workflows/staging-cleanup.yml vendored Normal file
View file

@ -0,0 +1,112 @@
name: Staging Cleanup
# Sweeps the production server for orphaned PR-preview resources whose PRs
# have closed without the cd-staging teardown job running (e.g., the
# teardown failed, the workflow file was changed mid-flight, or the PR was
# closed while runners were down). Runs weekly and can be triggered ad-hoc.
on:
schedule:
# Every Monday at 04:00 UTC
- cron: "0 4 * * 1"
workflow_dispatch: {}
concurrency:
group: staging-cleanup
cancel-in-progress: false
jobs:
cleanup:
name: Sweep orphaned previews
runs-on: ubuntu-latest
environment: production
permissions:
contents: read
pull-requests: read
steps:
- name: List active preview projects
id: list
uses: appleboy/ssh-action@v1
with:
host: ${{ secrets.DEPLOY_HOST }}
username: root
key: ${{ secrets.DEPLOY_SSH_KEY }}
script_stop: true
script: |
cd /opt/trails-cool
# Emit one project name per line, e.g. "trails-pr-123"
docker compose ls --format json --filter "name=trails-pr-" \
| python3 -c 'import json,sys
try:
data = json.load(sys.stdin)
except Exception:
data = []
for d in data:
n = d.get("Name","")
if n.startswith("trails-pr-"):
print(n)' \
|| true
- name: Determine which PRs are still open
id: orphans
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PROJECTS: ${{ steps.list.outputs.stdout }}
run: |
set -euo pipefail
ORPHANS=()
if [ -z "${PROJECTS:-}" ]; then
echo "No active preview projects."
echo "orphans=" >> "$GITHUB_OUTPUT"
exit 0
fi
while IFS= read -r project; do
[ -z "$project" ] && continue
pr="${project#trails-pr-}"
# If gh can't find the PR (deleted) or it's not OPEN, treat as orphan.
state=$(gh pr view "$pr" --repo "${{ github.repository }}" --json state -q .state 2>/dev/null || echo "MISSING")
if [ "$state" != "OPEN" ]; then
echo "Orphan: PR #$pr (state=$state) → tear down $project"
ORPHANS+=("$pr")
fi
done <<< "${PROJECTS}"
IFS=,
echo "orphans=${ORPHANS[*]:-}" >> "$GITHUB_OUTPUT"
- name: Tear down orphans
if: steps.orphans.outputs.orphans != ''
env:
ORPHANS: ${{ steps.orphans.outputs.orphans }}
uses: appleboy/ssh-action@v1
with:
host: ${{ secrets.DEPLOY_HOST }}
username: root
key: ${{ secrets.DEPLOY_SSH_KEY }}
envs: ORPHANS
script: |
set -euo pipefail
cd /opt/trails-cool
IFS=, read -ra PRS <<< "$ORPHANS"
for PR in "${PRS[@]}"; do
[ -z "$PR" ] && continue
PROJECT="trails-pr-$PR"
DB="trails_pr_$PR"
echo "→ tearing down $PROJECT"
# `down` needs the same env file the deploy used; staging.env on
# disk may belong to a different PR, so synthesize a minimal one.
cat > /tmp/cleanup.env <<EOF
DOMAIN=pr-$PR.staging.trails.cool
STAGING_DATABASE=$DB
JOURNAL_HOST_PORT=$((3200 + 2 * PR))
PLANNER_HOST_PORT=$((3201 + 2 * PR))
JWT_SECRET=cleanup
SESSION_SECRET=cleanup
BROUTER_URL=http://placeholder
BROUTER_AUTH_TOKEN=placeholder
EOF
docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file /tmp/cleanup.env down --remove-orphans || true
docker compose exec -T postgres dropdb -U trails --if-exists "$DB" || true
rm -f "sites/pr-$PR.caddyfile"
done
rm -f /tmp/cleanup.env
docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile || true

View file

@ -177,13 +177,15 @@ Admins can bypass the PR workflow when necessary (e.g., CI is broken and needs a
## Deployment
Three separate CD workflows triggered by path:
Five CD workflows triggered by path or event:
| Workflow | Triggers on | Deploys | Target |
|----------|-------------|---------|--------|
| `cd-apps.yml` | `apps/`, `packages/`, `pnpm-lock.yaml` | journal, planner | flagship (`root@trails.cool`) |
| `cd-infra.yml` | `infrastructure/` (except `brouter-host/**`) | caddy, postgres, prometheus, loki, grafana, exporters | flagship (`root@trails.cool`) |
| `cd-brouter.yml` | `docker/brouter/`, `infrastructure/brouter-host/**` | brouter + caddy sidecar | dedicated (`trails@ullrich.is:2232`) |
| `cd-staging.yml` | main push or PR open/sync/close on `apps/`, `packages/` | persistent staging + per-PR previews | flagship (alongside production) |
| `staging-cleanup.yml` | weekly cron + manual | sweeps orphaned PR previews | flagship |
### Hosts
@ -215,6 +217,38 @@ ssh -i ~/.ssh/trails-brouter-deploy -p 2232 trails@ullrich.is
### Grafana
`https://grafana.internal.trails.cool` — GitHub OAuth (trails-cool org)
### Staging & Previews
A persistent staging stack and ephemeral PR previews share the flagship server with production.
| Surface | URL | Database | Triggered by |
|---------|-----|----------|--------------|
| Persistent staging journal | `https://staging.trails.cool` | `trails_staging` | push to `main` |
| Persistent staging planner | `https://planner.staging.trails.cool` | `trails_staging` | push to `main` |
| PR preview journal | `https://pr-<N>.staging.trails.cool` | `trails_pr_<N>` | PR open/sync |
PR previews are **journal-only** — their `PLANNER_URL` points at the persistent staging planner so we don't pay 256MB per preview for an extra planner. The persistent staging planner's CSP allows `connect-src wss://*.staging.trails.cool` so PR-preview journals can talk to it.
**Port scheme** (host's loopback, reverse-proxied by Caddy via `host.docker.internal`):
- Persistent staging: journal `3100`, planner `3101`
- PR `<N>` preview: journal `3200 + 2N`, planner `3201 + 2N` (planner unused for previews)
**Compose project namespacing** keeps each preview isolated:
- Persistent staging: `-p trails-staging`
- PR `<N>`: `-p trails-pr-<N>`
The shared file `infrastructure/docker-compose.staging.yml` covers both — env vars (`DOMAIN`, `STAGING_DATABASE`, `JOURNAL_HOST_PORT`, `JOURNAL_IMAGE_TAG`, …) parametrize per target. Persistent staging uses `--profile persistent` to also start the planner; PR previews omit the profile.
**Caddy routing.** Persistent staging has fixed site blocks in `infrastructure/Caddyfile`. Per-PR site blocks are written by `cd-staging.yml` to `/opt/trails-cool/sites/pr-<N>.caddyfile` (mounted into Caddy at `/etc/caddy/sites/`) and picked up via `import sites/*.caddyfile` on a Caddy reload. No on-demand TLS; standard automatic HTTPS issues a per-host cert.
**Database isolation.** Each preview gets its own database on the production Postgres instance, schema applied via `drizzle-kit push --force`. Created on PR open, dropped on close. The persistent staging DB is never touched by previews.
**Concurrent preview cap.** Max 3 concurrent PR previews. When a 4th opens, the deploy job evicts the oldest project before deploying.
**Cleanup.** `cd-staging.yml`'s teardown job runs on PR close. `staging-cleanup.yml` runs weekly to catch orphans whose teardown never ran.
**Debugging.** SSH to the flagship (`ssh -i ~/.ssh/trails-cool-deploy root@trails.cool`) and run `docker compose -f docker-compose.staging.yml -p trails-pr-<N> logs -f` to tail a preview. `docker compose ls --filter name=trails-pr-` shows everything currently up.
## OpenSpec Workflow
Specs live in `openspec/`. Use these slash commands:

View file

@ -69,3 +69,52 @@ planner.{$DOMAIN:trails.cool} {
lb_try_interval 250ms
}
}
# ── Staging ──────────────────────────────────────────────────────────────
# Persistent staging instance. CSP allow-lists hardcode `staging.trails.cool`
# rather than `{$DOMAIN}` because the Caddy container runs with the
# production DOMAIN env (`trails.cool`); staging blocks need their own domain
# baked in. Upstreams are reached via `host.docker.internal` because the
# staging compose project publishes its containers on the host's loopback
# (127.0.0.1:3100/3101) rather than joining the production Caddy network.
staging.{$DOMAIN:trails.cool} {
import security_headers
import block_scanners
log {
output stdout
format json
}
reverse_proxy host.docker.internal:3100 {
lb_try_duration 30s
lb_try_interval 250ms
}
}
planner.staging.{$DOMAIN:trails.cool} {
header {
Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
X-Content-Type-Options "nosniff"
X-Frame-Options "DENY"
Referrer-Policy "strict-origin-when-cross-origin"
Permissions-Policy "camera=(), microphone=(), geolocation=()"
# connect-src includes wss + https://*.staging so PR-preview journals
# (pr-N.staging.trails.cool) can use this shared planner. PR previews
# are journal-only; this is the planner they all talk to.
Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' blob:; worker-src 'self' blob:; style-src 'self' 'unsafe-inline'; img-src 'self' data: https://*.tile.openstreetmap.org; connect-src 'self' wss://*.staging.{$DOMAIN:trails.cool} https://*.staging.{$DOMAIN:trails.cool} https://staging.{$DOMAIN:trails.cool} https://*.sentry.io https://*.ingest.de.sentry.io; font-src 'self';"
}
import block_scanners
log {
output stdout
format json
}
reverse_proxy host.docker.internal:3101 {
lb_try_duration 30s
lb_try_interval 250ms
}
}
# Per-PR preview snippets are written by cd-staging.yml into
# /etc/caddy/sites/pr-<N>.caddyfile and picked up here on Caddy reload. The
# glob is allowed to match nothing — Caddy treats an empty match as a no-op.
import /etc/caddy/sites/*.caddyfile

View file

@ -0,0 +1,100 @@
# Staging / PR-preview compose file.
#
# Used for both the persistent staging stack and ephemeral PR previews.
# The cd-staging.yml workflow picks the project name and fills in the host
# ports + DOMAIN + STAGING_DATABASE from the PR number (or "staging" for the
# persistent stack):
#
# # Persistent staging
# PROJECT=trails-staging
# JOURNAL_HOST_PORT=3100 PLANNER_HOST_PORT=3101 \
# DOMAIN=staging.trails.cool STAGING_DATABASE=trails_staging \
# docker compose -f docker-compose.staging.yml -p $PROJECT up -d
#
# # PR 123 preview (port = 3200 + 2N for journal, 3201 + 2N for planner)
# PROJECT=trails-pr-123
# JOURNAL_HOST_PORT=3446 PLANNER_HOST_PORT=3447 \
# DOMAIN=pr-123.staging.trails.cool STAGING_DATABASE=trails_pr_123 \
# JOURNAL_IMAGE_TAG=pr-123 PLANNER_IMAGE_TAG=pr-123 \
# docker compose -f docker-compose.staging.yml -p $PROJECT up -d
#
# `trails-shared` is created by the production compose file (see
# docker-compose.yml) so staging/preview containers can reach the production
# `postgres` host. BRouter lives on a separate host and is reached over
# vSwitch using the production-shared BROUTER_URL / BROUTER_AUTH_TOKEN.
#
# Container names are not pinned — Docker Compose generates them from the
# project name (e.g. `trails-pr-123-journal-1`), which keeps each preview
# isolated without manual naming.
services:
journal:
image: ghcr.io/trails-cool/journal:${JOURNAL_IMAGE_TAG:-latest}
restart: unless-stopped
ports:
- "127.0.0.1:${JOURNAL_HOST_PORT:?JOURNAL_HOST_PORT must be set}:3000"
networks:
- trails-shared
environment:
DOMAIN: ${DOMAIN:?DOMAIN must be set}
ORIGIN: https://${DOMAIN}
# PR previews override this to point at the persistent staging
# planner (planner.staging.trails.cool). Persistent staging defaults
# to its own planner subdomain.
PLANNER_URL: ${PLANNER_URL:-https://planner.${DOMAIN}}
IS_FLAGSHIP: ""
DATABASE_URL: postgres://trails:${POSTGRES_PASSWORD:-trails}@postgres:5432/${STAGING_DATABASE:?STAGING_DATABASE must be set}
JWT_SECRET: ${JWT_SECRET:?JWT_SECRET must be set}
SESSION_SECRET: ${SESSION_SECRET:?SESSION_SECRET must be set}
NODE_ENV: production
PORT: 3000
SENTRY_RELEASE: ${SENTRY_RELEASE:-}
SMTP_URL: ""
SMTP_FROM: trails.cool staging <noreply@staging.trails.cool>
WAHOO_CLIENT_ID: ""
WAHOO_CLIENT_SECRET: ""
WAHOO_WEBHOOK_TOKEN: ""
DEMO_BOT_ENABLED: ""
healthcheck:
test: ["CMD-SHELL", "curl -sf http://localhost:3000/api/health || exit 1"]
interval: 15s
timeout: 5s
retries: 3
deploy:
resources:
limits:
memory: 256M
planner:
image: ghcr.io/trails-cool/planner:${PLANNER_IMAGE_TAG:-latest}
# Only the persistent staging stack runs a planner. PR previews are
# journal-only and point their PLANNER_URL at the persistent
# planner.staging.trails.cool. Saves ~256MB per active preview.
profiles: ["persistent"]
restart: unless-stopped
ports:
- "127.0.0.1:${PLANNER_HOST_PORT:-3101}:3001"
networks:
- trails-shared
environment:
BROUTER_URL: ${BROUTER_URL:?BROUTER_URL must be set}
BROUTER_AUTH_TOKEN: ${BROUTER_AUTH_TOKEN:?BROUTER_AUTH_TOKEN must be set}
OVERPASS_URLS: ${OVERPASS_URLS:-https://lz4.overpass-api.de/api/interpreter,https://overpass-api.de/api/interpreter}
DATABASE_URL: postgres://trails:${POSTGRES_PASSWORD:-trails}@postgres:5432/${STAGING_DATABASE}
NODE_ENV: production
PORT: 3001
SENTRY_RELEASE: ${SENTRY_RELEASE:-}
healthcheck:
test: ["CMD-SHELL", "curl -sf http://localhost:3001/health || exit 1"]
interval: 15s
timeout: 5s
retries: 3
deploy:
resources:
limits:
memory: 256M
networks:
trails-shared:
external: true
name: trails-shared

View file

@ -6,10 +6,19 @@ services:
- "80:80"
- "443:443"
- "443:443/udp"
# host.docker.internal lets Caddy reverse-proxy to staging / PR-preview
# containers, which publish on the host's loopback (e.g. 127.0.0.1:3100).
# Linux requires `host-gateway` to make this name resolve.
extra_hosts:
- "host.docker.internal:host-gateway"
environment:
DOMAIN: ${DOMAIN:-trails.cool}
volumes:
- ./Caddyfile:/etc/caddy/Caddyfile:ro
# Per-PR staging snippets dropped in by cd-staging.yml. Bind mount
# auto-creates the host directory if it doesn't exist, so a fresh
# server doesn't need pre-provisioning.
- ./sites:/etc/caddy/sites:ro
- caddy_data:/data
- caddy_config:/config
depends_on:
@ -90,6 +99,12 @@ services:
postgres:
image: postgis/postgis:16-3.4
restart: unless-stopped
# Joined to the shared network so the staging compose project can reach
# this instance by hostname `postgres`. Production services keep using
# the default network as before.
networks:
- default
- trails-shared
environment:
POSTGRES_USER: trails
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-trails}
@ -226,6 +241,16 @@ services:
# - garage_data:/var/lib/garage
# - ./garage.toml:/etc/garage.toml:ro
networks:
# Default project network — kept implicit for production services.
default:
# Created by this compose project with a fixed (un-namespaced) name so the
# staging compose project can attach to it via `external: true`. Only
# postgres is joined here on the production side; staging containers join
# to reach postgres by hostname.
trails-shared:
name: trails-shared
volumes:
caddy_data:
caddy_config:

View file

@ -0,0 +1,29 @@
# Staging environment variables. The cd-staging.yml workflow assembles
# these from SOPS secrets + workflow-injected values and ships them to the
# server as `infrastructure/staging.env`. Keep this file in sync with the
# `environment:` blocks in `docker-compose.staging.yml`.
#
# For PR previews the same template is reused, with DOMAIN, STAGING_DATABASE,
# and image tags overridden per-PR by the workflow.
# Domain the staging stack serves at. Persistent staging uses
# staging.trails.cool; PR previews use pr-<n>.staging.trails.cool.
DOMAIN=staging.trails.cool
# Database name on the shared production Postgres. Persistent staging uses
# trails_staging; PR previews use trails_pr_<n>.
STAGING_DATABASE=trails_staging
# Inherited from SOPS secrets.app.env (decrypted in the workflow):
POSTGRES_PASSWORD=
JWT_SECRET=
SESSION_SECRET=
BROUTER_URL=
BROUTER_AUTH_TOKEN=
# Image tag to deploy. `latest` for staging, `pr-<n>` for PR previews.
JOURNAL_IMAGE_TAG=latest
PLANNER_IMAGE_TAG=latest
# Sentry release SHA, set by the workflow.
SENTRY_RELEASE=

View file

@ -191,6 +191,42 @@ resource "hcloud_zone_rrset" "planner_aaaa" {
records = [{ value = hcloud_server.trails.ipv6_address }]
}
# Staging persistent journal at staging.trails.cool, persistent planner
# at planner.staging.trails.cool, and PR previews at pr-<N>.staging.trails.cool
# (covered by the wildcard). All resolve to the flagship; Caddy routes per-host.
resource "hcloud_zone_rrset" "staging_a" {
zone = "trails.cool"
name = "staging"
type = "A"
ttl = 300
records = [{ value = hcloud_server.trails.ipv4_address }]
}
resource "hcloud_zone_rrset" "staging_aaaa" {
zone = "trails.cool"
name = "staging"
type = "AAAA"
ttl = 300
records = [{ value = hcloud_server.trails.ipv6_address }]
}
resource "hcloud_zone_rrset" "staging_wildcard_a" {
zone = "trails.cool"
name = "*.staging"
type = "A"
ttl = 300
records = [{ value = hcloud_server.trails.ipv4_address }]
}
resource "hcloud_zone_rrset" "staging_wildcard_aaaa" {
zone = "trails.cool"
name = "*.staging"
type = "AAAA"
ttl = 300
records = [{ value = hcloud_server.trails.ipv6_address }]
}
# Internal wildcard all *.internal.trails.cool resolves to the server.
# Caddy handles routing per-subdomain. No individual DNS entries needed
# for internal services (Grafana, Prometheus, etc.)

View file

@ -27,10 +27,10 @@ The server has headroom for lightweight additional containers. Caddy handles aut
**Rationale**: A separate Compose project gives clean namespace isolation (container names, volumes) without a second server. Sharing BRouter and monitoring avoids duplicating heavy services.
**Alternative considered**: Docker Compose profiles — simpler but risks accidental cross-contamination between production and staging in the same project.
### 2. Caddy on-demand TLS with wildcard routing
**Choice**: Use Caddy's `on_demand_tls` with a wildcard site block for `*.staging.trails.cool`. A small validation endpoint confirms which subdomains are active before Caddy obtains a certificate.
**Rationale**: Avoids pre-configuring Caddy for each PR. Caddy automatically provisions TLS certificates on first request. The validation endpoint prevents abuse (random subdomains triggering cert issuance).
**Alternative considered**: Wildcard certificate via DNS challenge — requires DNS API credentials and more complex setup.
### 2. Per-PR Caddyfile snippets with reload
**Choice**: The main Caddyfile uses `import sites/*.caddyfile`. The cd-staging workflow writes a snippet per PR (e.g. `sites/pr-123.caddyfile`) on PR open and removes it on PR close, then reloads Caddy in-place. Standard automatic HTTPS issues a per-host cert.
**Rationale**: Caddy can't compute upstream ports from a regex capture (the spec's `3200 + 2N` formula), so a wildcard block would need a sidecar router service. A graceful Caddy reload is fast and idempotent, and the existing cd-apps workflow already reloads Caddy on every deploy — extending that pattern is simpler than adding a new service.
**Alternatives considered**: (a) Wildcard `*.staging.trails.cool` block with on-demand TLS plus a small Node router that handles Caddy's `ask` check and wildcard host→port proxying — more moving parts. (b) DNS-challenge wildcard certificate — requires DNS API credentials in Caddy and more complex setup.
### 3. Per-PR databases in shared PostgreSQL
**Choice**: PR previews use per-PR databases (`trails_pr_123`) in the production PostgreSQL instance. Staging uses `trails_staging`. Created by the workflow, dropped on PR close.

View file

@ -1,7 +1,7 @@
## MODIFIED Requirements
### Requirement: Caddy reverse proxy routing
Caddy SHALL route requests to staging and PR preview containers via wildcard subdomain matching, in addition to the existing production routing.
Caddy SHALL route requests to staging and PR preview containers via per-host site blocks, in addition to the existing production routing.
#### Scenario: Staging subdomain routing
- **WHEN** a request arrives for `staging.trails.cool`
@ -13,12 +13,14 @@ Caddy SHALL route requests to staging and PR preview containers via wildcard sub
#### Scenario: PR preview routing
- **WHEN** a request arrives for `pr-123.staging.trails.cool`
- **THEN** Caddy proxies it to the PR 123 journal container on the correct dynamically assigned port
- **THEN** Caddy proxies it to the PR 123 journal container on its assigned port (`3200 + 2N`)
#### Scenario: On-demand TLS for staging subdomains
- **WHEN** a first request arrives for a new staging subdomain
- **THEN** Caddy automatically provisions a TLS certificate via Let's Encrypt
- **AND** a validation endpoint confirms the subdomain is an active staging/preview environment before certificate issuance
#### Scenario: Per-PR Caddyfile snippet lifecycle
- **WHEN** a PR preview is deployed
- **THEN** the cd-staging workflow writes a Caddyfile snippet at `sites/pr-<N>.caddyfile` and reloads Caddy
- **WHEN** a PR is closed
- **THEN** the workflow removes the snippet and reloads Caddy
- **AND** standard automatic HTTPS issues / retains the per-host certificate via Let's Encrypt
### Requirement: Docker Compose deployment
The staging environment SHALL be deployed as a separate Docker Compose project alongside production on the same server.

View file

@ -1,38 +1,38 @@
## 1. DNS & TLS Setup
- [ ] 1.1 Add wildcard DNS record `*.staging.trails.cool` pointing to the Hetzner server IP
- [ ] 1.2 Add `staging.trails.cool` and `planner.staging.trails.cool` DNS A records
- [x] 1.1 Add wildcard DNS record `*.staging.trails.cool` pointing to the Hetzner server IP
- [x] 1.2 Add `staging.trails.cool` and `planner.staging.trails.cool` DNS A records (planner.staging covered by the wildcard)
## 2. Docker Compose Staging Configuration
- [ ] 2.1 Create `infrastructure/docker-compose.staging.yml` with staging journal (port 3100), planner (port 3101), memory limits (256MB), and `trails_staging` database URL
- [ ] 2.2 Create `infrastructure/staging.env.template` documenting required staging environment variables (DOMAIN, DATABASE_URL, JWT_SECRET, SESSION_SECRET)
- [ ] 2.3 Add a shared Docker network (`trails-shared`) to production `docker-compose.yml` so staging can reach BRouter and PostgreSQL
- [x] 2.1 Create `infrastructure/docker-compose.staging.yml` with staging journal (port 3100), planner (port 3101), memory limits (256MB), and `trails_staging` database URL
- [x] 2.2 Create `infrastructure/staging.env.template` documenting required staging environment variables (DOMAIN, DATABASE_URL, JWT_SECRET, SESSION_SECRET)
- [x] 2.3 Add a shared Docker network (`trails-shared`) to production `docker-compose.yml` so staging can reach BRouter and PostgreSQL
- [ ] 2.4 Verify staging containers start with `docker compose -f docker-compose.staging.yml -p trails-staging up -d` on the server
## 3. Caddy Wildcard Routing
- [ ] 3.1 Add `staging.trails.cool` site block proxying to journal on port 3100
- [ ] 3.2 Add `planner.staging.trails.cool` site block proxying to planner on port 3101
- [ ] 3.3 Add `*.staging.trails.cool` wildcard site block with on-demand TLS for PR previews — extract PR number from subdomain, proxy to `localhost:3200 + (PR * 2)`
- [ ] 3.4 Create a TLS validation endpoint (small script or Caddy matcher) that checks if the requested subdomain corresponds to a running container
- [x] 3.1 Add `staging.trails.cool` site block proxying to journal on port 3100
- [x] 3.2 Add `planner.staging.trails.cool` site block proxying to planner on port 3101
- [x] 3.3 Add `import sites/*.caddyfile` to the main Caddyfile so per-PR site blocks can be dropped in and picked up on reload
- [x] 3.4 Define the per-PR Caddyfile snippet template the cd-staging workflow writes for each preview (host = `pr-<N>.staging.trails.cool`, upstream = `host.docker.internal:<port>`, port = `3200 + 2N`)
- [ ] 3.5 Reload Caddy and verify staging routes work with `curl -sf https://staging.trails.cool/api/health`
## 4. GitHub Actions Workflow
- [ ] 4.1 Create `.github/workflows/cd-staging.yml` triggered on push to main (paths: `apps/`, `packages/`) and on PR open/synchronize/close (same paths)
- [ ] 4.2 Implement the **staging deploy** job: build images, SSH to server, `docker compose -f docker-compose.staging.yml -p trails-staging pull && up -d`, run Drizzle push against `trails_staging`
- [ ] 4.3 Implement the **PR preview deploy** job: compute ports from PR number, create `trails_pr_<number>` database if not exists, build images tagged with PR number, deploy containers, post preview URL as PR comment
- [ ] 4.4 Implement the **PR preview teardown** job: stop and remove PR containers, drop `trails_pr_<number>` database, delete PR comment
- [ ] 4.5 Add the concurrent preview limit check: if >3 active previews, tear down the oldest before deploying a new one
- [x] 4.1 Create `.github/workflows/cd-staging.yml` triggered on push to main (paths: `apps/`, `packages/`) and on PR open/synchronize/close (same paths)
- [x] 4.2 Implement the **staging deploy** job: build images, SSH to server, `docker compose -f docker-compose.staging.yml -p trails-staging pull && up -d`, run Drizzle push against `trails_staging`
- [x] 4.3 Implement the **PR preview deploy** job: compute ports from PR number, create `trails_pr_<number>` database if not exists, build images tagged with PR number, deploy containers, post preview URL as PR comment
- [x] 4.4 Implement the **PR preview teardown** job: stop and remove PR containers, drop `trails_pr_<number>` database, delete PR comment
- [x] 4.5 Add the concurrent preview limit check: if >3 active previews, tear down the oldest before deploying a new one
## 5. Cleanup & Safety
- [ ] 5.1 Create a scheduled cleanup job (weekly cron in GitHub Actions or pg-boss on the server) that lists running `trails-pr-*` containers, checks PR status via `gh pr view`, and tears down orphans
- [ ] 5.2 Add memory limits (`deploy.resources.limits.memory: 256m`) to staging containers in the compose override
- [x] 5.1 Create a scheduled cleanup job (weekly cron in GitHub Actions or pg-boss on the server) that lists running `trails-pr-*` containers, checks PR status via `gh pr view`, and tears down orphans
- [x] 5.2 Add memory limits (`deploy.resources.limits.memory: 256m`) to staging containers in the compose override
- [ ] 5.3 Test full lifecycle: open a test PR → verify preview deploys → push a commit → verify preview updates → close PR → verify teardown
## 6. Documentation
- [ ] 6.1 Add a "Staging & Previews" section to CLAUDE.md documenting the staging URL, PR preview URL pattern, port scheme, and how to debug staging issues
- [ ] 6.2 Update the Deployment table in CLAUDE.md with the new `cd-staging.yml` workflow
- [x] 6.1 Add a "Staging & Previews" section to CLAUDE.md documenting the staging URL, PR preview URL pattern, port scheme, and how to debug staging issues
- [x] 6.2 Update the Deployment table in CLAUDE.md with the new `cd-staging.yml` workflow