The journal/planner deploy in cd-apps.yml does `docker compose up -d journal planner`, which stops the old container and starts the new one — Caddy keeps forwarding requests during the ~10–30s gap and returns 502s. The caddy-502-rate alert (threshold > 0 for 2m) correctly trips, every time. Two production changes plus a long-broken workflow detail: - infrastructure/Caddyfile — add `lb_try_duration 30s` / `lb_try_interval 250ms` to the journal and planner reverse_proxy blocks. Caddy now holds and retries the upstream for up to 30s during a restart instead of 502'ing immediately. Real outages (upstream unreachable longer than 30s) still 502 and the alert still fires for those. - infrastructure/grafana/provisioning/alerting/alerts.yml — add a comment documenting why caddy-502-rate stays at threshold > 0: with lb_try_duration in front of it, the alert no longer conflates "deploy in flight" with "real outage." - .github/workflows/cd-apps.yml — fix a long-silent bug: the Grafana deploy-annotation step was reading GRAFANA_SERVICE_TOKEN from `.env`, but the secrets file we scp to /opt/trails-cool is named `app.env`. The token check failed silently and the curl was being skipped on every deploy. Switching to `app.env` so deploys actually annotate Grafana. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
132 lines
4.8 KiB
YAML
132 lines
4.8 KiB
YAML
name: CD Apps
|
|
|
|
on:
|
|
push:
|
|
branches: [main]
|
|
paths:
|
|
- "apps/**"
|
|
- "packages/**"
|
|
- "pnpm-lock.yaml"
|
|
workflow_dispatch: {}
|
|
|
|
concurrency:
|
|
group: deploy-apps
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
build-images:
|
|
name: Build & Push Docker Images
|
|
runs-on: ubuntu-latest
|
|
environment: production
|
|
permissions:
|
|
contents: read
|
|
packages: write
|
|
strategy:
|
|
matrix:
|
|
app: [journal, planner]
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
|
|
- uses: docker/login-action@v4
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Decrypt Sentry auth token
|
|
run: |
|
|
curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64
|
|
chmod +x sops-v3.9.4.linux.amd64
|
|
SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > /tmp/secrets.env
|
|
grep SENTRY_AUTH_TOKEN /tmp/secrets.env | cut -d= -f2- | tr -d '\n' > /tmp/sentry_token
|
|
|
|
- uses: docker/build-push-action@v7
|
|
with:
|
|
context: .
|
|
file: apps/${{ matrix.app }}/Dockerfile
|
|
push: true
|
|
tags: |
|
|
ghcr.io/trails-cool/${{ matrix.app }}:latest
|
|
ghcr.io/trails-cool/${{ matrix.app }}:${{ github.sha }}
|
|
build-args: |
|
|
SENTRY_RELEASE=${{ github.sha }}
|
|
secrets: |
|
|
SENTRY_AUTH_TOKEN=/tmp/sentry_token
|
|
|
|
deploy:
|
|
name: Deploy Apps
|
|
needs: [build-images]
|
|
runs-on: ubuntu-latest
|
|
environment: production
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
|
|
- name: Decrypt secrets
|
|
run: |
|
|
curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64
|
|
chmod +x sops-v3.9.4.linux.amd64
|
|
SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > infrastructure/app.env
|
|
echo "SENTRY_RELEASE=${{ github.sha }}" >> infrastructure/app.env
|
|
echo "DOMAIN=trails.cool" >> infrastructure/app.env
|
|
# Flagship marker — see cd-infra.yml for what this gates.
|
|
echo "IS_FLAGSHIP=true" >> infrastructure/app.env
|
|
|
|
- name: Copy files to server
|
|
uses: appleboy/scp-action@v1
|
|
with:
|
|
host: ${{ secrets.DEPLOY_HOST }}
|
|
username: root
|
|
key: ${{ secrets.DEPLOY_SSH_KEY }}
|
|
source: "infrastructure/docker-compose.yml,infrastructure/Caddyfile"
|
|
target: /opt/trails-cool
|
|
strip_components: 1
|
|
|
|
- name: Copy secrets to server
|
|
uses: appleboy/scp-action@v1
|
|
with:
|
|
host: ${{ secrets.DEPLOY_HOST }}
|
|
username: root
|
|
key: ${{ secrets.DEPLOY_SSH_KEY }}
|
|
source: "infrastructure/app.env"
|
|
target: /opt/trails-cool
|
|
strip_components: 1
|
|
|
|
- name: Deploy via SSH
|
|
uses: appleboy/ssh-action@v1
|
|
with:
|
|
host: ${{ secrets.DEPLOY_HOST }}
|
|
username: root
|
|
key: ${{ secrets.DEPLOY_SSH_KEY }}
|
|
script: |
|
|
cd /opt/trails-cool
|
|
|
|
# Login to ghcr.io
|
|
GHCR_TOKEN=$(grep DEPLOY_GHCR_TOKEN app.env | cut -d= -f2-)
|
|
echo "$GHCR_TOKEN" | docker login ghcr.io -u stigi --password-stdin
|
|
|
|
# Pull and deploy app containers
|
|
docker compose --env-file app.env pull journal planner
|
|
docker compose --env-file app.env run --rm journal npx drizzle-kit push --config /app/packages/db/drizzle.config.ts --force
|
|
# --remove-orphans cleans up containers whose service was deleted
|
|
# from the compose file, matching cd-infra's behaviour.
|
|
docker compose --env-file app.env up -d --remove-orphans journal planner
|
|
|
|
# Clean up
|
|
docker image prune -af
|
|
docker compose ps
|
|
|
|
# Annotate deploy in Grafana. The token lives in the
|
|
# decrypted SOPS env file we just scp'd to /opt/trails-cool
|
|
# — that file is `app.env`, not `.env`. (Pre-fix this read
|
|
# the wrong path, so annotations were silently no-op'ing
|
|
# every deploy.) `2>/dev/null` keeps a missing token from
|
|
# failing the deploy; `|| true` keeps the curl from
|
|
# failing the deploy if Grafana itself is unhealthy.
|
|
GRAFANA_TOKEN=$(grep GRAFANA_SERVICE_TOKEN app.env 2>/dev/null | cut -d= -f2-)
|
|
if [ -n "$GRAFANA_TOKEN" ]; then
|
|
docker compose exec -T grafana curl -sf -X POST \
|
|
-H "Authorization: Bearer $GRAFANA_TOKEN" \
|
|
-H "Content-Type: application/json" \
|
|
-d '{"text":"Deploy ${{ github.sha }}","tags":["deploy","apps"]}' \
|
|
http://localhost:3000/api/annotations || true
|
|
fi
|