From a75eb1965ca73d6fcdead61b5cfb61ac42e2acd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= Date: Tue, 14 Jul 2026 09:27:50 +0200 Subject: [PATCH] ci: don't fail deploys on a transient "prune already running" collision cd-apps's post-deploy `docker image prune` runs under `set -euo pipefail` after the containers are already swapped, so when it collides with a concurrent deploy/disk-maintenance prune ("Error response from daemon: a prune operation is already running") it fails an otherwise-successful deploy (observed on the #587 planner deploy). The cleanup is best-effort; disk-maintenance.yml is the real image-prune safety net. - cd-apps: `docker image prune -af || true`. - disk-maintenance: tolerate the same collision on its prune; the disk-% threshold check afterward stays the real failure gate. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/cd-apps.yml | 9 +++++++-- .github/workflows/disk-maintenance.yml | 6 +++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cd-apps.yml b/.github/workflows/cd-apps.yml index 8f7d794..a5a967e 100644 --- a/.github/workflows/cd-apps.yml +++ b/.github/workflows/cd-apps.yml @@ -177,8 +177,13 @@ jobs: # deploy from failing if Caddy itself is unhealthy. docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile || true - # Clean up - docker image prune -af + # Clean up (best-effort). This runs AFTER the containers are + # swapped + Caddy reloaded, so the deploy already succeeded — + # a transient "a prune operation is already running" collision + # with a concurrent deploy/disk-maintenance prune must NOT fail + # an otherwise-green deploy. disk-maintenance.yml is the real + # image-prune safety net. + docker image prune -af || true docker compose ps # Annotate deploy in Grafana. GRAFANA_SERVICE_TOKEN lives diff --git a/.github/workflows/disk-maintenance.yml b/.github/workflows/disk-maintenance.yml index 5113a0c..d8f6a09 100644 --- a/.github/workflows/disk-maintenance.yml +++ b/.github/workflows/disk-maintenance.yml @@ -39,7 +39,11 @@ jobs: echo "before: $(df -h / | tail -1)" # 12h filter: never touch layers a same-day deploy may still # be assembling; running containers' images are never pruned. - docker image prune -af --filter "until=12h" + # `|| true`: a concurrent deploy's prune can make this collide + # with "a prune operation is already running" — that's benign + # (the other prune is freeing space too), and the disk-% gate + # below is the real assertion, so don't fail on the collision. + docker image prune -af --filter "until=12h" || true echo "after: $(df -h / | tail -1)" PCT=$(df --output=pcent / | tail -1 | tr -dc '0-9')