diff --git a/docs/deployment.md b/docs/deployment.md index 1edecf6..5add95c 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -67,6 +67,31 @@ gh workflow run cd-infra.yml -f restart_all=true Restarts every flagship service. Does NOT touch the BRouter host. +## Flagship disk pressure + +The flagship root fs is ~38 GB. When it hits 100%, Postgres can't write and +every app 500s; deploys fail at the SCP step ("error copy file to dest"). +Two independent guards keep it in check: + +- **Image churn** — `disk-maintenance.yml` prunes unused images daily (04:30 + UTC, `until=12h`); the deploy workflows also prune after each run. +- **Container logs** — every compose service caps its `json-file` logs at + 30 MB (`x-logging` anchor in `docker-compose.yml`). Before that cap a + single service logging in a loop grew a 2.6 GB log (2026-07-14 outage). + +Emergency (disk already full, service down): + +```bash +ssh -i ~/.ssh/trails-cool-deploy root@trails.cool +df -h / # confirm 100% +docker image prune -af # safe — never removes volumes/data +docker builder prune -f # build cache +docker system df # what's left; du -xhd1 /var to hunt +# containers self-recover once space frees; re-run the failed deploy after. +``` + +Never `docker system prune --volumes` — that deletes the Postgres data volume. + ## Network-changing deploys (flagship) Changing options on an existing Docker network (`enable_ipv6`, subnets, diff --git a/infrastructure/docker-compose.yml b/infrastructure/docker-compose.yml index 3374bd9..4280115 100644 --- a/infrastructure/docker-compose.yml +++ b/infrastructure/docker-compose.yml @@ -1,7 +1,18 @@ +# Cap every container's json-file logs so a service logging in a loop can't +# fill the flagship disk. 10 MB x 3 rotated files = 30 MB max per container. +# (2026-07-14: an unbounded 2.6 GB container log helped push / to 100% and +# took production down — see docs/deployment.md.) +x-logging: &default-logging + driver: json-file + options: + max-size: "10m" + max-file: "3" + services: caddy: image: caddy:2 restart: unless-stopped + logging: *default-logging ports: - "80:80" - "443:443" @@ -35,6 +46,7 @@ services: journal: image: ghcr.io/trails-cool/journal:latest restart: unless-stopped + logging: *default-logging environment: DOMAIN: ${DOMAIN:-trails.cool} ORIGIN: https://${DOMAIN:-trails.cool} @@ -93,6 +105,7 @@ services: planner: image: ghcr.io/trails-cool/planner:latest restart: unless-stopped + logging: *default-logging environment: # Required: points the Planner at the dedicated BRouter host over # vSwitch. Set in SOPS `secrets.app.env` (BROUTER_URL line). Using @@ -120,6 +133,7 @@ services: postgres: image: postgis/postgis:16-3.4 restart: unless-stopped + logging: *default-logging # Joined to the shared network so the staging compose project can reach # this instance by hostname `postgres`. Production services keep using # the default network as before. @@ -148,6 +162,7 @@ services: postgres-exporter: image: prometheuscommunity/postgres-exporter:latest restart: unless-stopped + logging: *default-logging environment: DATA_SOURCE_NAME: postgresql://trails:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in SOPS secrets.app.env}@postgres:5432/trails?sslmode=disable PG_EXPORTER_EXTEND_QUERY_PATH: /etc/postgres-exporter/queries.yml @@ -160,6 +175,7 @@ services: node-exporter: image: prom/node-exporter:latest restart: unless-stopped + logging: *default-logging pid: host volumes: - /proc:/host/proc:ro @@ -174,6 +190,7 @@ services: cadvisor: image: gcr.io/cadvisor/cadvisor:latest restart: unless-stopped + logging: *default-logging privileged: true volumes: - /:/rootfs:ro @@ -185,6 +202,7 @@ services: promtail: image: grafana/promtail:latest restart: unless-stopped + logging: *default-logging volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - /var/lib/docker/containers:/var/lib/docker/containers:ro @@ -198,6 +216,7 @@ services: prometheus: image: prom/prometheus:latest restart: unless-stopped + logging: *default-logging volumes: # Directory mount (not the file) so config changes are seen live and a # SIGHUP reload applies them without recreating the container — see the @@ -212,6 +231,7 @@ services: loki: image: grafana/loki:latest restart: unless-stopped + logging: *default-logging volumes: # Directory mount (not the file) so a replaced config is seen live — # see the caddy note above. Loki only reloads its main config on @@ -229,6 +249,7 @@ services: grafana: image: grafana/grafana:latest restart: unless-stopped + logging: *default-logging environment: GF_SERVER_ROOT_URL: https://grafana.internal.${DOMAIN:-trails.cool} GF_AUTH_GITHUB_ENABLED: "true"