Merge branch 'main' into ci/opt-in-pr-previews

apps/planner/app/lib/yjs-server.test.ts

@@ -52,9 +52,12 @@ describe("countActiveSessions", () => {
 });
 
 describe("doc-size caps", () => {
-  it("MAX_MESSAGE_BYTES + MAX_DOC_BYTES are positive and ordered sanely", () => {
-    expect(MAX_MESSAGE_BYTES).toBeGreaterThan(0);
-    expect(MAX_DOC_BYTES).toBeGreaterThan(MAX_MESSAGE_BYTES);
+  it("the per-frame cap can carry a full-doc sync (message cap >= doc cap)", () => {
+    expect(MAX_DOC_BYTES).toBeGreaterThan(0);
+    // A full-state sync frame contains the entire doc, so a per-frame cap
+    // below the doc cap makes docs between the two sizes unsyncable (reconnect
+    // loop). The frame cap must be at least the doc cap, plus protocol room.
+    expect(MAX_MESSAGE_BYTES).toBeGreaterThanOrEqual(MAX_DOC_BYTES);
   });
 
   it("docByteSize returns 0 for an unknown session", () => {

apps/planner/app/lib/yjs-server.ts

@@ -11,13 +11,19 @@ import { plannerActiveSessions, plannerConnectedClients } from "./metrics.server
 const messageSync = 0;
 const messageAwareness = 1;
 
-// Bound the per-WebSocket-message size and per-session doc size. Both
-// guards exist to stop a single connected client from filling memory
-// (or the persisted `yjs_state` blob) without limit. Picked generously
-// — a multi-day route with hundreds of waypoints is well under 100 KB
-// of serialized Yjs state — but small enough to refuse abuse.
-export const MAX_MESSAGE_BYTES = 256 * 1024; // 256 KB per WS frame
+// Bound the per-session doc size — the real abuse guard, stopping a client
+// from filling memory or the persisted `yjs_state` blob without limit. A
+// route with hundreds of waypoints is well under this once its BRouter
+// geometry is stored in the doc, which is what actually drives the size.
 export const MAX_DOC_BYTES = 5 * 1024 * 1024; // 5 MB per session doc
+// Per-WebSocket-frame cap. It MUST exceed the doc cap: a full-state sync
+// (initial load, or any reconnect) puts the entire doc into a single frame,
+// so a frame limit below MAX_DOC_BYTES makes any legal-but-larger-than-the-
+// -frame doc permanently unsyncable — the client's sync frame is rejected,
+// it reconnects, resends the same frame, and loops forever (observed at the
+// old 256 KB cap: a 305 KB / 4-waypoint route froze in a reconnect storm).
+// The doc cap remains the size limit; this just leaves room to transmit it.
+export const MAX_MESSAGE_BYTES = MAX_DOC_BYTES + 256 * 1024; // doc + protocol overhead
 // WebSocket close code for policy-violation responses (1008 is the
 // standard RFC 6455 code for "received a message that violates policy").
 const POLICY_VIOLATION = 1008;

docs/deployment.md

@@ -67,6 +67,31 @@ gh workflow run cd-infra.yml -f restart_all=true
 
 Restarts every flagship service. Does NOT touch the BRouter host.
 
+## Flagship disk pressure
+
+The flagship root fs is ~38 GB. When it hits 100%, Postgres can't write and
+every app 500s; deploys fail at the SCP step ("error copy file to dest").
+Two independent guards keep it in check:
+
+- **Image churn** — `disk-maintenance.yml` prunes unused images daily (04:30
+  UTC, `until=12h`); the deploy workflows also prune after each run.
+- **Container logs** — every compose service caps its `json-file` logs at
+  30 MB (`x-logging` anchor in `docker-compose.yml`). Before that cap a
+  single service logging in a loop grew a 2.6 GB log (2026-07-14 outage).
+
+Emergency (disk already full, service down):
+
+```bash
+ssh -i ~/.ssh/trails-cool-deploy root@trails.cool
+df -h /                                  # confirm 100%
+docker image prune -af                   # safe — never removes volumes/data
+docker builder prune -f                  # build cache
+docker system df                         # what's left; du -xhd1 /var to hunt
+# containers self-recover once space frees; re-run the failed deploy after.
+```
+
+Never `docker system prune --volumes` — that deletes the Postgres data volume.
+
 ## Network-changing deploys (flagship)
 
 Changing options on an existing Docker network (`enable_ipv6`, subnets,

infrastructure/docker-compose.yml

@@ -1,7 +1,18 @@
+# Cap every container's json-file logs so a service logging in a loop can't
+# fill the flagship disk. 10 MB x 3 rotated files = 30 MB max per container.
+# (2026-07-14: an unbounded 2.6 GB container log helped push / to 100% and
+# took production down — see docs/deployment.md.)
+x-logging: &default-logging
+  driver: json-file
+  options:
+    max-size: "10m"
+    max-file: "3"
+
 services:
   caddy:
     image: caddy:2
     restart: unless-stopped
+    logging: *default-logging
     ports:
       - "80:80"
       - "443:443"
@@ -35,6 +46,7 @@ services:
   journal:
     image: ghcr.io/trails-cool/journal:latest
     restart: unless-stopped
+    logging: *default-logging
     environment:
       DOMAIN: ${DOMAIN:-trails.cool}
       ORIGIN: https://${DOMAIN:-trails.cool}
@@ -93,6 +105,7 @@ services:
   planner:
     image: ghcr.io/trails-cool/planner:latest
     restart: unless-stopped
+    logging: *default-logging
     environment:
       # Required: points the Planner at the dedicated BRouter host over
       # vSwitch. Set in SOPS `secrets.app.env` (BROUTER_URL line). Using
@@ -120,6 +133,7 @@ services:
   postgres:
     image: postgis/postgis:16-3.4
     restart: unless-stopped
+    logging: *default-logging
     # Joined to the shared network so the staging compose project can reach
     # this instance by hostname `postgres`. Production services keep using
     # the default network as before.
@@ -148,6 +162,7 @@ services:
   postgres-exporter:
     image: prometheuscommunity/postgres-exporter:latest
     restart: unless-stopped
+    logging: *default-logging
     environment:
       DATA_SOURCE_NAME: postgresql://trails:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in SOPS secrets.app.env}@postgres:5432/trails?sslmode=disable
       PG_EXPORTER_EXTEND_QUERY_PATH: /etc/postgres-exporter/queries.yml
@@ -160,6 +175,7 @@ services:
   node-exporter:
     image: prom/node-exporter:latest
     restart: unless-stopped
+    logging: *default-logging
     pid: host
     volumes:
       - /proc:/host/proc:ro
@@ -174,6 +190,7 @@ services:
   cadvisor:
     image: gcr.io/cadvisor/cadvisor:latest
     restart: unless-stopped
+    logging: *default-logging
     privileged: true
     volumes:
       - /:/rootfs:ro
@@ -185,6 +202,7 @@ services:
   promtail:
     image: grafana/promtail:latest
     restart: unless-stopped
+    logging: *default-logging
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
@@ -198,6 +216,7 @@ services:
   prometheus:
     image: prom/prometheus:latest
     restart: unless-stopped
+    logging: *default-logging
     volumes:
       # Directory mount (not the file) so config changes are seen live and a
       # SIGHUP reload applies them without recreating the container — see the
@@ -212,6 +231,7 @@ services:
   loki:
     image: grafana/loki:latest
     restart: unless-stopped
+    logging: *default-logging
     volumes:
       # Directory mount (not the file) so a replaced config is seen live —
       # see the caddy note above. Loki only reloads its main config on
@@ -229,6 +249,7 @@ services:
   grafana:
     image: grafana/grafana:latest
     restart: unless-stopped
+    logging: *default-logging
     environment:
       GF_SERVER_ROOT_URL: https://grafana.internal.${DOMAIN:-trails.cool}
       GF_AUTH_GITHUB_ENABLED: "true"