From e23751f29e36273fff1952550b706f1ac55f17ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= Date: Sun, 10 May 2026 13:03:52 +0200 Subject: [PATCH 1/3] Fix concurrent preview deploy race: per-PR env files + server flock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three changes: 1. Skip GH-Actions-only Dependabot PRs (dependabot/github_actions/*) in deploy-preview — there is no app image to preview, and these PRs were landing phantom containers with mismatched ports. 2. Use per-PR env files (staging-pr-{N}.env) instead of the shared staging.env for preview deploys and teardowns. Concurrent SCP transfers to the same filename were overwriting each other, causing wrong JOURNAL_HOST_PORT / JOURNAL_IMAGE_TAG values to be used. 3. Serialize server-side deploy operations with a flock on /tmp/trails-preview-deploy.lock (300s timeout). Eviction + compose up must be atomic; without the lock, two simultaneous jobs could both see "3 active previews" and both evict different projects, or one could start compose up against a just-evicted env. Triggered by a Dependabot batch today (PRs 371-373) that opened simultaneously and produced a phantom trails-pr-371 container running the pr-370 image on port 3940 while the Caddyfile expected 3942, causing sustained 502s on pr-371.staging.trails.cool. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/cd-staging.yml | 52 +++++++++++++++++---------- .github/workflows/staging-cleanup.yml | 2 +- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/.github/workflows/cd-staging.yml b/.github/workflows/cd-staging.yml index 8956ab9..ec5091a 100644 --- a/.github/workflows/cd-staging.yml +++ b/.github/workflows/cd-staging.yml @@ -177,7 +177,11 @@ jobs: # ── PR preview deploy ──────────────────────────────────────────────── deploy-preview: name: Deploy PR Preview - if: github.event_name == 'pull_request' && github.event.action != 'closed' + # Skip GH-Actions-only Dependabot PRs — there is no app image to preview. + if: > + github.event_name == 'pull_request' && + github.event.action != 'closed' && + !startsWith(github.head_ref, 'dependabot/github_actions/') needs: [build-images] runs-on: ubuntu-latest environment: production @@ -204,7 +208,8 @@ jobs: run: | curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64 chmod +x sops-v3.9.4.linux.amd64 - SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > infrastructure/staging.env + # Use a per-PR filename so concurrent SCP transfers don't overwrite each other. + SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > infrastructure/staging-pr-${{ steps.ports.outputs.pr }}.env { echo "DOMAIN=${{ steps.ports.outputs.host }}" echo "STAGING_DATABASE=${{ steps.ports.outputs.database }}" @@ -215,7 +220,7 @@ jobs: # PR-preview journals all share the persistent staging planner. echo "PLANNER_URL=https://planner.staging.trails.cool" echo "SENTRY_RELEASE=${{ github.event.pull_request.head.sha }}" - } >> infrastructure/staging.env + } >> infrastructure/staging-pr-${{ steps.ports.outputs.pr }}.env - name: Generate per-PR Caddyfile snippet run: | @@ -242,7 +247,7 @@ jobs: host: ${{ secrets.DEPLOY_HOST }} username: root key: ${{ secrets.DEPLOY_SSH_KEY }} - source: "infrastructure/docker-compose.staging.yml,infrastructure/staging.env,infrastructure/sites/pr-${{ steps.ports.outputs.pr }}.caddyfile" + source: "infrastructure/docker-compose.staging.yml,infrastructure/staging-pr-${{ steps.ports.outputs.pr }}.env,infrastructure/sites/pr-${{ steps.ports.outputs.pr }}.caddyfile" target: /opt/trails-cool strip_components: 1 @@ -252,18 +257,25 @@ jobs: PR: ${{ steps.ports.outputs.pr }} PROJECT: ${{ steps.ports.outputs.project }} DB: ${{ steps.ports.outputs.database }} + JOURNAL_PORT: ${{ steps.ports.outputs.journal_port }} with: host: ${{ secrets.DEPLOY_HOST }} username: root key: ${{ secrets.DEPLOY_SSH_KEY }} - envs: PR,PROJECT,DB + envs: PR,PROJECT,DB,JOURNAL_PORT script: | set -euo pipefail cd /opt/trails-cool + ENV_FILE="staging-pr-${PR}.env" - GHCR_TOKEN=$(grep DEPLOY_GHCR_TOKEN staging.env | cut -d= -f2-) + GHCR_TOKEN=$(grep DEPLOY_GHCR_TOKEN "$ENV_FILE" | cut -d= -f2-) echo "$GHCR_TOKEN" | docker login ghcr.io -u stigi --password-stdin + # Serialize all preview deploys with a server-side lock so concurrent + # CI jobs (e.g. a Dependabot batch) can't race on eviction or compose state. + exec 9>/tmp/trails-preview-deploy.lock + flock --timeout 300 9 || { echo "Timed out waiting for deploy lock after 300s"; exit 1; } + # Same network bootstrap as deploy-staging — see comment there. docker network inspect trails-shared >/dev/null 2>&1 || docker network create trails-shared PG_CONTAINER=$(docker ps --filter "name=trails-cool-postgres" --format '{{.Names}}' | head -1) @@ -286,9 +298,12 @@ jobs: if [ -n "$OLDEST" ] && [ "$OLDEST" != "$PROJECT" ]; then echo "At cap; evicting oldest preview: $OLDEST" OLD_PR=${OLDEST#trails-pr-} - docker compose -f docker-compose.staging.yml -p "$OLDEST" --env-file staging.env down --remove-orphans || true + OLD_ENV="staging-pr-${OLD_PR}.env" + # Fall back to base secrets if the per-PR env was already cleaned up. + EVICT_ENV=$( [ -f "$OLD_ENV" ] && echo "$OLD_ENV" || echo "staging.env" ) + docker compose -f docker-compose.staging.yml -p "$OLDEST" --env-file "$EVICT_ENV" down --remove-orphans || true docker compose exec -T postgres dropdb -U trails --if-exists "trails_pr_$OLD_PR" || true - rm -f "sites/pr-$OLD_PR.caddyfile" + rm -f "sites/pr-$OLD_PR.caddyfile" "$OLD_ENV" fi fi fi @@ -303,14 +318,14 @@ jobs: "CREATE EXTENSION IF NOT EXISTS postgis" # Pull, migrate, deploy (journal-only — no --profile means planner skipped) - docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file staging.env pull journal - docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file staging.env run --rm journal npx drizzle-kit push --config /app/packages/db/drizzle.config.ts --force - docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file staging.env up -d --remove-orphans journal + docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file "$ENV_FILE" pull journal + docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file "$ENV_FILE" run --rm journal npx drizzle-kit push --config /app/packages/db/drizzle.config.ts --force + docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file "$ENV_FILE" up -d --remove-orphans journal # Reload Caddy to pick up the per-PR snippet (writes/replaces it from the SCP step) docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile - docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file staging.env ps + docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file "$ENV_FILE" ps # Find any prior preview comment so we can update it in place rather # than spamming a new one each push. The marker line at the bottom of @@ -364,13 +379,13 @@ jobs: run: | curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64 chmod +x sops-v3.9.4.linux.amd64 - SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > infrastructure/staging.env + SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > infrastructure/staging-pr-${{ steps.ports.outputs.pr }}.env { echo "DOMAIN=pr-${{ steps.ports.outputs.pr }}.staging.trails.cool" echo "STAGING_DATABASE=${{ steps.ports.outputs.database }}" echo "JOURNAL_HOST_PORT=$((3200 + 2 * ${{ steps.ports.outputs.pr }}))" echo "PLANNER_HOST_PORT=$((3201 + 2 * ${{ steps.ports.outputs.pr }}))" - } >> infrastructure/staging.env + } >> infrastructure/staging-pr-${{ steps.ports.outputs.pr }}.env - name: Copy compose + env (teardown still needs the file) uses: appleboy/scp-action@v1 @@ -378,7 +393,7 @@ jobs: host: ${{ secrets.DEPLOY_HOST }} username: root key: ${{ secrets.DEPLOY_SSH_KEY }} - source: "infrastructure/docker-compose.staging.yml,infrastructure/staging.env" + source: "infrastructure/docker-compose.staging.yml,infrastructure/staging-pr-${{ steps.ports.outputs.pr }}.env" target: /opt/trails-cool strip_components: 1 @@ -396,15 +411,16 @@ jobs: script: | set -euo pipefail cd /opt/trails-cool + ENV_FILE="staging-pr-${PR}.env" # Stop and remove containers + volumes for this PR - docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file staging.env down --remove-orphans || true + docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file "$ENV_FILE" down --remove-orphans || true # Drop the per-PR database (idempotent) docker compose exec -T postgres dropdb -U trails --if-exists "$DB" || true - # Remove the per-PR Caddy snippet and reload - rm -f "sites/pr-$PR.caddyfile" + # Remove the per-PR Caddy snippet, env file, and reload + rm -f "sites/pr-$PR.caddyfile" "$ENV_FILE" docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile || true - name: Find existing preview comment diff --git a/.github/workflows/staging-cleanup.yml b/.github/workflows/staging-cleanup.yml index 595888f..bed2fd4 100644 --- a/.github/workflows/staging-cleanup.yml +++ b/.github/workflows/staging-cleanup.yml @@ -106,7 +106,7 @@ jobs: EOF docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file /tmp/cleanup.env down --remove-orphans || true docker compose exec -T postgres dropdb -U trails --if-exists "$DB" || true - rm -f "sites/pr-$PR.caddyfile" + rm -f "sites/pr-$PR.caddyfile" "staging-pr-${PR}.env" done rm -f /tmp/cleanup.env docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile || true From 8f635fd5d1c7221fffcfe234cb2cbee704af4e2b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 10 May 2026 11:11:45 +0000 Subject: [PATCH 2/3] Bump peter-evans/create-or-update-comment from 4 to 5 Bumps [peter-evans/create-or-update-comment](https://github.com/peter-evans/create-or-update-comment) from 4 to 5. - [Release notes](https://github.com/peter-evans/create-or-update-comment/releases) - [Commits](https://github.com/peter-evans/create-or-update-comment/compare/v4...v5) --- updated-dependencies: - dependency-name: peter-evans/create-or-update-comment dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/cd-staging.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd-staging.yml b/.github/workflows/cd-staging.yml index ec5091a..3fe1ac1 100644 --- a/.github/workflows/cd-staging.yml +++ b/.github/workflows/cd-staging.yml @@ -339,7 +339,7 @@ jobs: body-includes: "" - name: Upsert preview comment on PR - uses: peter-evans/create-or-update-comment@v4 + uses: peter-evans/create-or-update-comment@v5 with: comment-id: ${{ steps.find-comment.outputs.comment-id }} issue-number: ${{ github.event.number }} @@ -433,7 +433,7 @@ jobs: - name: Update preview comment on close if: steps.find-comment.outputs.comment-id - uses: peter-evans/create-or-update-comment@v4 + uses: peter-evans/create-or-update-comment@v5 with: comment-id: ${{ steps.find-comment.outputs.comment-id }} edit-mode: replace From 0e8978452a163583d4235ab342c4d6a656ac3b6b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 10 May 2026 11:11:47 +0000 Subject: [PATCH 3/3] Bump peter-evans/find-comment from 3 to 4 Bumps [peter-evans/find-comment](https://github.com/peter-evans/find-comment) from 3 to 4. - [Release notes](https://github.com/peter-evans/find-comment/releases) - [Commits](https://github.com/peter-evans/find-comment/compare/v3...v4) --- updated-dependencies: - dependency-name: peter-evans/find-comment dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/cd-staging.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd-staging.yml b/.github/workflows/cd-staging.yml index ec5091a..adb890d 100644 --- a/.github/workflows/cd-staging.yml +++ b/.github/workflows/cd-staging.yml @@ -331,7 +331,7 @@ jobs: # than spamming a new one each push. The marker line at the bottom of # the body is what `body-includes` matches on. - name: Find existing preview comment - uses: peter-evans/find-comment@v3 + uses: peter-evans/find-comment@v4 id: find-comment with: issue-number: ${{ github.event.number }} @@ -424,7 +424,7 @@ jobs: docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile || true - name: Find existing preview comment - uses: peter-evans/find-comment@v3 + uses: peter-evans/find-comment@v4 id: find-comment with: issue-number: ${{ github.event.number }}