diff --git a/apps/planner/app/lib/yjs-server.test.ts b/apps/planner/app/lib/yjs-server.test.ts index e5eb4f4..308d0e6 100644 --- a/apps/planner/app/lib/yjs-server.test.ts +++ b/apps/planner/app/lib/yjs-server.test.ts @@ -52,9 +52,12 @@ describe("countActiveSessions", () => { }); describe("doc-size caps", () => { - it("MAX_MESSAGE_BYTES + MAX_DOC_BYTES are positive and ordered sanely", () => { - expect(MAX_MESSAGE_BYTES).toBeGreaterThan(0); - expect(MAX_DOC_BYTES).toBeGreaterThan(MAX_MESSAGE_BYTES); + it("the per-frame cap can carry a full-doc sync (message cap >= doc cap)", () => { + expect(MAX_DOC_BYTES).toBeGreaterThan(0); + // A full-state sync frame contains the entire doc, so a per-frame cap + // below the doc cap makes docs between the two sizes unsyncable (reconnect + // loop). The frame cap must be at least the doc cap, plus protocol room. + expect(MAX_MESSAGE_BYTES).toBeGreaterThanOrEqual(MAX_DOC_BYTES); }); it("docByteSize returns 0 for an unknown session", () => { diff --git a/apps/planner/app/lib/yjs-server.ts b/apps/planner/app/lib/yjs-server.ts index de48bfb..f917812 100644 --- a/apps/planner/app/lib/yjs-server.ts +++ b/apps/planner/app/lib/yjs-server.ts @@ -11,13 +11,19 @@ import { plannerActiveSessions, plannerConnectedClients } from "./metrics.server const messageSync = 0; const messageAwareness = 1; -// Bound the per-WebSocket-message size and per-session doc size. Both -// guards exist to stop a single connected client from filling memory -// (or the persisted `yjs_state` blob) without limit. Picked generously -// — a multi-day route with hundreds of waypoints is well under 100 KB -// of serialized Yjs state — but small enough to refuse abuse. -export const MAX_MESSAGE_BYTES = 256 * 1024; // 256 KB per WS frame +// Bound the per-session doc size — the real abuse guard, stopping a client +// from filling memory or the persisted `yjs_state` blob without limit. A +// route with hundreds of waypoints is well under this once its BRouter +// geometry is stored in the doc, which is what actually drives the size. export const MAX_DOC_BYTES = 5 * 1024 * 1024; // 5 MB per session doc +// Per-WebSocket-frame cap. It MUST exceed the doc cap: a full-state sync +// (initial load, or any reconnect) puts the entire doc into a single frame, +// so a frame limit below MAX_DOC_BYTES makes any legal-but-larger-than-the- +// -frame doc permanently unsyncable — the client's sync frame is rejected, +// it reconnects, resends the same frame, and loops forever (observed at the +// old 256 KB cap: a 305 KB / 4-waypoint route froze in a reconnect storm). +// The doc cap remains the size limit; this just leaves room to transmit it. +export const MAX_MESSAGE_BYTES = MAX_DOC_BYTES + 256 * 1024; // doc + protocol overhead // WebSocket close code for policy-violation responses (1008 is the // standard RFC 6455 code for "received a message that violates policy"). const POLICY_VIOLATION = 1008; diff --git a/docs/deployment.md b/docs/deployment.md index 1edecf6..5add95c 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -67,6 +67,31 @@ gh workflow run cd-infra.yml -f restart_all=true Restarts every flagship service. Does NOT touch the BRouter host. +## Flagship disk pressure + +The flagship root fs is ~38 GB. When it hits 100%, Postgres can't write and +every app 500s; deploys fail at the SCP step ("error copy file to dest"). +Two independent guards keep it in check: + +- **Image churn** — `disk-maintenance.yml` prunes unused images daily (04:30 + UTC, `until=12h`); the deploy workflows also prune after each run. +- **Container logs** — every compose service caps its `json-file` logs at + 30 MB (`x-logging` anchor in `docker-compose.yml`). Before that cap a + single service logging in a loop grew a 2.6 GB log (2026-07-14 outage). + +Emergency (disk already full, service down): + +```bash +ssh -i ~/.ssh/trails-cool-deploy root@trails.cool +df -h / # confirm 100% +docker image prune -af # safe — never removes volumes/data +docker builder prune -f # build cache +docker system df # what's left; du -xhd1 /var to hunt +# containers self-recover once space frees; re-run the failed deploy after. +``` + +Never `docker system prune --volumes` — that deletes the Postgres data volume. + ## Network-changing deploys (flagship) Changing options on an existing Docker network (`enable_ipv6`, subnets, diff --git a/infrastructure/docker-compose.yml b/infrastructure/docker-compose.yml index 3374bd9..4280115 100644 --- a/infrastructure/docker-compose.yml +++ b/infrastructure/docker-compose.yml @@ -1,7 +1,18 @@ +# Cap every container's json-file logs so a service logging in a loop can't +# fill the flagship disk. 10 MB x 3 rotated files = 30 MB max per container. +# (2026-07-14: an unbounded 2.6 GB container log helped push / to 100% and +# took production down — see docs/deployment.md.) +x-logging: &default-logging + driver: json-file + options: + max-size: "10m" + max-file: "3" + services: caddy: image: caddy:2 restart: unless-stopped + logging: *default-logging ports: - "80:80" - "443:443" @@ -35,6 +46,7 @@ services: journal: image: ghcr.io/trails-cool/journal:latest restart: unless-stopped + logging: *default-logging environment: DOMAIN: ${DOMAIN:-trails.cool} ORIGIN: https://${DOMAIN:-trails.cool} @@ -93,6 +105,7 @@ services: planner: image: ghcr.io/trails-cool/planner:latest restart: unless-stopped + logging: *default-logging environment: # Required: points the Planner at the dedicated BRouter host over # vSwitch. Set in SOPS `secrets.app.env` (BROUTER_URL line). Using @@ -120,6 +133,7 @@ services: postgres: image: postgis/postgis:16-3.4 restart: unless-stopped + logging: *default-logging # Joined to the shared network so the staging compose project can reach # this instance by hostname `postgres`. Production services keep using # the default network as before. @@ -148,6 +162,7 @@ services: postgres-exporter: image: prometheuscommunity/postgres-exporter:latest restart: unless-stopped + logging: *default-logging environment: DATA_SOURCE_NAME: postgresql://trails:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in SOPS secrets.app.env}@postgres:5432/trails?sslmode=disable PG_EXPORTER_EXTEND_QUERY_PATH: /etc/postgres-exporter/queries.yml @@ -160,6 +175,7 @@ services: node-exporter: image: prom/node-exporter:latest restart: unless-stopped + logging: *default-logging pid: host volumes: - /proc:/host/proc:ro @@ -174,6 +190,7 @@ services: cadvisor: image: gcr.io/cadvisor/cadvisor:latest restart: unless-stopped + logging: *default-logging privileged: true volumes: - /:/rootfs:ro @@ -185,6 +202,7 @@ services: promtail: image: grafana/promtail:latest restart: unless-stopped + logging: *default-logging volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - /var/lib/docker/containers:/var/lib/docker/containers:ro @@ -198,6 +216,7 @@ services: prometheus: image: prom/prometheus:latest restart: unless-stopped + logging: *default-logging volumes: # Directory mount (not the file) so config changes are seen live and a # SIGHUP reload applies them without recreating the container — see the @@ -212,6 +231,7 @@ services: loki: image: grafana/loki:latest restart: unless-stopped + logging: *default-logging volumes: # Directory mount (not the file) so a replaced config is seen live — # see the caddy note above. Loki only reloads its main config on @@ -229,6 +249,7 @@ services: grafana: image: grafana/grafana:latest restart: unless-stopped + logging: *default-logging environment: GF_SERVER_ROOT_URL: https://grafana.internal.${DOMAIN:-trails.cool} GF_AUTH_GITHUB_ENABLED: "true"