diff --git a/.github/workflows/cd-staging.yml b/.github/workflows/cd-staging.yml index 7b87971..ed59b70 100644 --- a/.github/workflows/cd-staging.yml +++ b/.github/workflows/cd-staging.yml @@ -195,6 +195,13 @@ jobs: # via cd-infra) are live. Idempotent. docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile || true + # Reclaim superseded image layers. cd-apps prunes after its own + # deploys, but a day of staging/preview deploys while cd-apps is + # red can fill the disk on its own (2026-06-07: 100% full, + # postgres down). The 1h filter avoids racing layers another + # in-flight deploy just pulled. + docker image prune -af --filter "until=1h" || true + docker compose -f docker-compose.staging.yml -p trails-staging --env-file staging.env --profile persistent ps # ── PR preview deploy ──────────────────────────────────────────────── @@ -356,6 +363,10 @@ jobs: # Reload Caddy to pick up the per-PR snippet (writes/replaces it from the SCP step) docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile + # Same disk-hygiene prune as the persistent staging deploy — + # preview pushes are the highest-volume image source. + docker image prune -af --filter "until=1h" || true + docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file "$ENV_FILE" ps # Find any prior preview comment so we can update it in place rather diff --git a/.github/workflows/disk-maintenance.yml b/.github/workflows/disk-maintenance.yml new file mode 100644 index 0000000..5113a0c --- /dev/null +++ b/.github/workflows/disk-maintenance.yml @@ -0,0 +1,52 @@ +name: Disk Maintenance + +# Daily safety net for flagship disk usage. The deploy workflows prune +# superseded image layers after their own runs, but that protection +# disappears exactly when it's needed most: a deploy that fails early +# never reaches its prune step, while other workflows keep pulling +# fresh images (2026-06-07 incident: cd-apps red all day, ~10 staging +# deploys, disk 100% full, postgres down on a Saturday morning). +# +# Also doubles as a redundant alert channel: the run FAILS when the +# disk is still above the threshold after pruning, so a scheduled-run +# failure email lands even if the Grafana disk alert drowns in other +# noise (which is what happened during the incident). + +on: + schedule: + # Daily at 04:30 UTC (offset from staging-cleanup's Monday 04:00) + - cron: "30 4 * * *" + workflow_dispatch: {} + +concurrency: + group: disk-maintenance + cancel-in-progress: false + +jobs: + prune-flagship: + name: Prune unused images (flagship) + runs-on: ubuntu-latest + environment: production + steps: + - name: Prune and check disk headroom + uses: appleboy/ssh-action@v1 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: root + key: ${{ secrets.DEPLOY_SSH_KEY }} + script: | + set -euo pipefail + echo "before: $(df -h / | tail -1)" + # 12h filter: never touch layers a same-day deploy may still + # be assembling; running containers' images are never pruned. + docker image prune -af --filter "until=12h" + echo "after: $(df -h / | tail -1)" + + PCT=$(df --output=pcent / | tail -1 | tr -dc '0-9') + if [ "$PCT" -ge 85 ]; then + echo "Disk still at ${PCT}% after pruning — needs a human." + echo "Largest docker consumers:" + docker system df + exit 1 + fi + echo "Disk at ${PCT}% — healthy."