From 89caedca058a7bb17901b2cb8e1986fa5de30eb9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= <hi+github@ullrich.is>
Date: Sun, 7 Jun 2026 09:52:49 +0200
Subject: [PATCH] ci: scheduled image prune + staging-deploy disk hygiene
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The 2026-06-07 disk-full outage (flagship 100%, postgres crash-looping
on its pidfile, prod + staging down): cd-apps DOES prune images after
deploys, but it had been failing early all day (migration bug), so its
prune never ran — while ~10 staging/preview deploys kept pulling fresh
images with no prune of their own.

- cd-staging: prune superseded layers (until=1h guard against racing
  in-flight pulls) after persistent staging and preview deploys.
- disk-maintenance.yml: NEW daily scheduled prune (04:30 UTC) that
  also FAILS when the disk is still ≥85% after pruning — a redundant
  alert channel for exactly the case where the Grafana disk alert
  drowns in other noise, as it did during the incident.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .github/workflows/cd-staging.yml       | 11 ++++++
 .github/workflows/disk-maintenance.yml | 52 ++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)
 create mode 100644 .github/workflows/disk-maintenance.yml

diff --git a/.github/workflows/cd-staging.yml b/.github/workflows/cd-staging.yml
index 7b87971..ed59b70 100644
--- a/.github/workflows/cd-staging.yml
+++ b/.github/workflows/cd-staging.yml
@@ -195,6 +195,13 @@ jobs:
             # via cd-infra) are live. Idempotent.
             docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile || true
 
+            # Reclaim superseded image layers. cd-apps prunes after its own
+            # deploys, but a day of staging/preview deploys while cd-apps is
+            # red can fill the disk on its own (2026-06-07: 100% full,
+            # postgres down). The 1h filter avoids racing layers another
+            # in-flight deploy just pulled.
+            docker image prune -af --filter "until=1h" || true
+
             docker compose -f docker-compose.staging.yml -p trails-staging --env-file staging.env --profile persistent ps
 
   # ── PR preview deploy ────────────────────────────────────────────────
@@ -356,6 +363,10 @@ jobs:
             # Reload Caddy to pick up the per-PR snippet (writes/replaces it from the SCP step)
             docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile
 
+            # Same disk-hygiene prune as the persistent staging deploy —
+            # preview pushes are the highest-volume image source.
+            docker image prune -af --filter "until=1h" || true
+
             docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file "$ENV_FILE" ps
 
       # Find any prior preview comment so we can update it in place rather
diff --git a/.github/workflows/disk-maintenance.yml b/.github/workflows/disk-maintenance.yml
new file mode 100644
index 0000000..5113a0c
--- /dev/null
+++ b/.github/workflows/disk-maintenance.yml
@@ -0,0 +1,52 @@
+name: Disk Maintenance
+
+# Daily safety net for flagship disk usage. The deploy workflows prune
+# superseded image layers after their own runs, but that protection
+# disappears exactly when it's needed most: a deploy that fails early
+# never reaches its prune step, while other workflows keep pulling
+# fresh images (2026-06-07 incident: cd-apps red all day, ~10 staging
+# deploys, disk 100% full, postgres down on a Saturday morning).
+#
+# Also doubles as a redundant alert channel: the run FAILS when the
+# disk is still above the threshold after pruning, so a scheduled-run
+# failure email lands even if the Grafana disk alert drowns in other
+# noise (which is what happened during the incident).
+
+on:
+  schedule:
+    # Daily at 04:30 UTC (offset from staging-cleanup's Monday 04:00)
+    - cron: "30 4 * * *"
+  workflow_dispatch: {}
+
+concurrency:
+  group: disk-maintenance
+  cancel-in-progress: false
+
+jobs:
+  prune-flagship:
+    name: Prune unused images (flagship)
+    runs-on: ubuntu-latest
+    environment: production
+    steps:
+      - name: Prune and check disk headroom
+        uses: appleboy/ssh-action@v1
+        with:
+          host: ${{ secrets.DEPLOY_HOST }}
+          username: root
+          key: ${{ secrets.DEPLOY_SSH_KEY }}
+          script: |
+            set -euo pipefail
+            echo "before: $(df -h / | tail -1)"
+            # 12h filter: never touch layers a same-day deploy may still
+            # be assembling; running containers' images are never pruned.
+            docker image prune -af --filter "until=12h"
+            echo "after:  $(df -h / | tail -1)"
+
+            PCT=$(df --output=pcent / | tail -1 | tr -dc '0-9')
+            if [ "$PCT" -ge 85 ]; then
+              echo "Disk still at ${PCT}% after pruning — needs a human."
+              echo "Largest docker consumers:"
+              docker system df
+              exit 1
+            fi
+            echo "Disk at ${PCT}% — healthy."