trails/infrastructure/docker-compose.yml
Ullrich Schäfer 010edae636
fix(infra): cap container log sizes to prevent disk-full outages
Every compose service used docker's default json-file driver with no
rotation, so a service logging in a loop grew unbounded — a 2.6 GB
container log contributed to a 2026-07-14 disk-full outage (/ at 100%,
Postgres down, apps 500ing, deploys failing at the SCP step).

Add an `x-logging` anchor (10 MB x 3 files = 30 MB max per container) and
apply it to all 11 services. Complements disk-maintenance.yml (which prunes
images but can't touch container logs). Takes effect when cd-infra recreates
the containers. Documented the disk-pressure runbook in docs/deployment.md.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-14 08:34:17 +02:00

320 lines
13 KiB
YAML

# Cap every container's json-file logs so a service logging in a loop can't
# fill the flagship disk. 10 MB x 3 rotated files = 30 MB max per container.
# (2026-07-14: an unbounded 2.6 GB container log helped push / to 100% and
# took production down — see docs/deployment.md.)
x-logging: &default-logging
driver: json-file
options:
max-size: "10m"
max-file: "3"
services:
caddy:
image: caddy:2
restart: unless-stopped
logging: *default-logging
ports:
- "80:80"
- "443:443"
- "443:443/udp"
# host.docker.internal lets Caddy reverse-proxy to staging / PR-preview
# containers, which publish on the host's loopback (e.g. 127.0.0.1:3100).
# Linux requires `host-gateway` to make this name resolve.
extra_hosts:
- "host.docker.internal:host-gateway"
environment:
DOMAIN: ${DOMAIN:-trails.cool}
volumes:
# Mount the caddy/ DIRECTORY, not the Caddyfile directly. A single-
# file bind mount pins to the host file's inode at create time, so a
# deploy that replaces the file (scp) leaves the container reading the
# old inode — `caddy reload` then reloads stale config. A directory
# mount resolves children live, so reload picks up the new Caddyfile.
# The container path /etc/caddy/Caddyfile is unchanged.
- ./caddy:/etc/caddy:ro
# Per-PR staging snippets dropped in by cd-staging.yml, overlaid onto
# the read-only caddy dir above (caddy/sites/.gitkeep keeps the
# mountpoint present). Bind mount auto-creates the host directory if
# it doesn't exist, so a fresh server doesn't need pre-provisioning.
- ./sites:/etc/caddy/sites:ro
- caddy_data:/data
- caddy_config:/config
depends_on:
- journal
- planner
journal:
image: ghcr.io/trails-cool/journal:latest
restart: unless-stopped
logging: *default-logging
environment:
DOMAIN: ${DOMAIN:-trails.cool}
ORIGIN: https://${DOMAIN:-trails.cool}
PLANNER_URL: https://planner.${DOMAIN:-trails.cool}
# "true" only on the flagship trails.cool instance. The Journal
# home renders the project marketing block when set, and the
# "Powered by trails.cool" footer link when not. Default empty =
# self-host, which is the safer default.
IS_FLAGSHIP: ${IS_FLAGSHIP:-}
DATABASE_URL: postgres://trails:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in SOPS secrets.app.env}@postgres:5432/trails
JWT_SECRET: ${JWT_SECRET:?JWT_SECRET must be set in SOPS secrets.app.env}
SESSION_SECRET: ${SESSION_SECRET:?SESSION_SECRET must be set in SOPS secrets.app.env}
NODE_ENV: production
PORT: 3000
# SENTRY_DSN per service (journal + planner have separate Sentry
# projects on the same org). Empty = no Sentry init (correct
# default for self-hosted instances). See cd-apps.yml for how the
# flagship populates this.
SENTRY_DSN: ${SENTRY_DSN_JOURNAL:-}
SENTRY_RELEASE: ${SENTRY_RELEASE:-}
SMTP_URL: ${SMTP_URL:-}
SMTP_FROM: ${SMTP_FROM:-trails.cool <noreply@trails.cool>}
WAHOO_CLIENT_ID: ${WAHOO_CLIENT_ID:-}
WAHOO_CLIENT_SECRET: ${WAHOO_CLIENT_SECRET:-}
WAHOO_WEBHOOK_TOKEN: ${WAHOO_WEBHOOK_TOKEN:-}
# Garmin Connect Developer Program keys (spec: garmin-import).
# Empty = the Garmin provider is hidden on /settings/connections.
GARMIN_CLIENT_ID: ${GARMIN_CLIENT_ID:-}
GARMIN_CLIENT_SECRET: ${GARMIN_CLIENT_SECRET:-}
# Federation (ActivityPub, social-federation rollout 12.5). Empty
# = off: every federation surface 404s — the safe self-host
# default. The flagship turns it on via cd-apps.yml (same pattern
# as IS_FLAGSHIP); the encryption key comes from SOPS. Rollback is
# dropping the flag and redeploying — instant, follow rows
# persist. See docs/deployment.md "Federation runbook".
FEDERATION_ENABLED: ${FEDERATION_ENABLED:-}
FEDERATION_KEY_ENCRYPTION_KEY: ${FEDERATION_KEY_ENCRYPTION_KEY:-}
FEDERATION_LOG_LEVEL: ${FEDERATION_LOG_LEVEL:-}
INTEGRATION_SECRET: ${INTEGRATION_SECRET:?INTEGRATION_SECRET must be set in SOPS secrets.app.env}
# Demo-activity-bot. Disabled by default everywhere; flip to "true"
# only in prod. When unset, DEMO_BOT_PERSONA uses the built-in
# Bruno/Berlin persona. See docs/demo-persona.md for the schema.
DEMO_BOT_ENABLED: ${DEMO_BOT_ENABLED:-}
DEMO_BOT_RETENTION_DAYS: ${DEMO_BOT_RETENTION_DAYS:-}
DEMO_BOT_REGION: ${DEMO_BOT_REGION:-}
DEMO_BOT_PERSONA: ${DEMO_BOT_PERSONA:-}
healthcheck:
test: ["CMD-SHELL", "curl -sf http://localhost:3000/api/health || exit 1"]
interval: 15s
timeout: 5s
retries: 3
depends_on:
postgres:
condition: service_healthy
planner:
image: ghcr.io/trails-cool/planner:latest
restart: unless-stopped
logging: *default-logging
environment:
# Required: points the Planner at the dedicated BRouter host over
# vSwitch. Set in SOPS `secrets.app.env` (BROUTER_URL line). Using
# `:?` so a missing value fails the compose up loudly rather than
# silently pointing at a nonexistent `brouter:17777` service.
BROUTER_URL: ${BROUTER_URL:?BROUTER_URL must be set in SOPS secrets.app.env}
# Shared secret the Planner attaches as X-BRouter-Auth on every
# BRouter request. The Caddy sidecar on the dedicated host
# enforces this header. Set in SOPS secrets.app.env.
BROUTER_AUTH_TOKEN: ${BROUTER_AUTH_TOKEN:?BROUTER_AUTH_TOKEN must be set in SOPS secrets.app.env}
DATABASE_URL: postgres://trails:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in SOPS secrets.app.env}@postgres:5432/trails
NODE_ENV: production
PORT: 3001
SENTRY_DSN: ${SENTRY_DSN_PLANNER:-}
SENTRY_RELEASE: ${SENTRY_RELEASE:-}
healthcheck:
test: ["CMD-SHELL", "curl -sf http://localhost:3001/health || exit 1"]
interval: 15s
timeout: 5s
retries: 3
depends_on:
postgres:
condition: service_healthy
postgres:
image: postgis/postgis:16-3.4
restart: unless-stopped
logging: *default-logging
# Joined to the shared network so the staging compose project can reach
# this instance by hostname `postgres`. Production services keep using
# the default network as before.
networks:
- default
- trails-shared
environment:
POSTGRES_USER: trails
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in SOPS secrets.app.env}
POSTGRES_DB: trails
volumes:
- pgdata:/var/lib/postgresql/data
- ./postgres/init-grafana-user.sql:/docker-entrypoint-initdb.d/init-grafana-user.sql:ro
command:
- "postgres"
- "-c"
- "shared_preload_libraries=pg_stat_statements"
- "-c"
- "pg_stat_statements.track=all"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U trails"]
interval: 5s
timeout: 5s
retries: 5
postgres-exporter:
image: prometheuscommunity/postgres-exporter:latest
restart: unless-stopped
logging: *default-logging
environment:
DATA_SOURCE_NAME: postgresql://trails:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in SOPS secrets.app.env}@postgres:5432/trails?sslmode=disable
PG_EXPORTER_EXTEND_QUERY_PATH: /etc/postgres-exporter/queries.yml
volumes:
- ./postgres/queries.yml:/etc/postgres-exporter/queries.yml:ro
depends_on:
postgres:
condition: service_healthy
node-exporter:
image: prom/node-exporter:latest
restart: unless-stopped
logging: *default-logging
pid: host
volumes:
- /proc:/host/proc:ro
- /sys:/host/sys:ro
- /:/rootfs:ro
command:
- "--path.procfs=/host/proc"
- "--path.sysfs=/host/sys"
- "--path.rootfs=/rootfs"
- "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
cadvisor:
image: gcr.io/cadvisor/cadvisor:latest
restart: unless-stopped
logging: *default-logging
privileged: true
volumes:
- /:/rootfs:ro
- /var/run:/var/run:ro
- /sys:/sys:ro
- /var/lib/docker/:/var/lib/docker:ro
- /dev/disk/:/dev/disk:ro
promtail:
image: grafana/promtail:latest
restart: unless-stopped
logging: *default-logging
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
- /var/lib/docker/containers:/var/lib/docker/containers:ro
# Directory mount (not the file) so a deploy's replaced config is seen
# live and a SIGHUP reload picks it up — see the caddy note above.
- ./promtail:/etc/promtail:ro
command: ["-config.file=/etc/promtail/promtail-config.yml"]
depends_on:
- loki
prometheus:
image: prom/prometheus:latest
restart: unless-stopped
logging: *default-logging
volumes:
# Directory mount (not the file) so config changes are seen live and a
# SIGHUP reload applies them without recreating the container — see the
# caddy note above. --config.file path below is unchanged.
- ./prometheus:/etc/prometheus:ro
- prometheus_data:/prometheus
command:
- "--config.file=/etc/prometheus/prometheus.yml"
- "--storage.tsdb.retention.time=15d"
- "--storage.tsdb.retention.size=2GB"
loki:
image: grafana/loki:latest
restart: unless-stopped
logging: *default-logging
volumes:
# Directory mount (not the file) so a replaced config is seen live —
# see the caddy note above. Loki only reloads its main config on
# restart (no SIGHUP), so cd-infra restarts loki rather than HUPs it.
- ./loki:/etc/loki:ro
- loki_data:/loki
command: ["-config.file=/etc/loki/loki-config.yml"]
# Publish only on the vSwitch IP so Promtail running on the
# dedicated BRouter host can push logs in. Hetzner Cloud firewall
# still blocks 3100 from the public internet. Internal services on
# this host reach Loki via the docker network as before.
ports:
- "10.0.0.2:3100:3100"
grafana:
image: grafana/grafana:latest
restart: unless-stopped
logging: *default-logging
environment:
GF_SERVER_ROOT_URL: https://grafana.internal.${DOMAIN:-trails.cool}
GF_AUTH_GITHUB_ENABLED: "true"
GF_AUTH_GITHUB_CLIENT_ID: ${GF_AUTH_GITHUB_CLIENT_ID:-}
GF_AUTH_GITHUB_CLIENT_SECRET: ${GF_AUTH_GITHUB_CLIENT_SECRET:-}
GF_AUTH_GITHUB_ALLOWED_ORGANIZATIONS: trails-cool
GF_AUTH_GITHUB_SCOPES: user:email,read:org
GF_AUTH_OAUTH_ALLOW_INSECURE_EMAIL_LOOKUP: "true"
GF_AUTH_DISABLE_LOGIN_FORM: "true"
# New GitHub-authenticated users land as Editor, not the default
# Viewer, so they can use Explore (ad-hoc Loki/Prometheus queries)
# without an admin promotion step. Access is already gated by the
# GF_AUTH_GITHUB_ALLOWED_ORGANIZATIONS allowlist above.
GF_USERS_AUTO_ASSIGN_ORG_ROLE: Editor
GF_SMTP_ENABLED: "true"
GF_SMTP_HOST: ${GF_SMTP_HOST:-}
GF_SMTP_USER: ${GF_SMTP_USER:-}
GF_SMTP_PASSWORD: ${GF_SMTP_PASSWORD:-}
GF_SMTP_FROM_ADDRESS: noreply@trails.cool
GF_SMTP_FROM_NAME: trails.cool Grafana
GRAFANA_DB_PASSWORD: ${GRAFANA_DB_PASSWORD:-}
# Pushover app + user keys consumed by alerts.yml provisioning.
# App token identifies the trails.cool Pushover application; per-user
# keys receive the notifications. Keys (un)set in SOPS secrets.infra.env.
PUSHOVER_API_TOKEN: ${PUSHOVER_API_TOKEN:-}
PUSHOVER_USER_KEY_ULLRICH: ${PUSHOVER_USER_KEY_ULLRICH:-}
volumes:
- ./grafana/provisioning:/etc/grafana/provisioning:ro
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana_data:/var/lib/grafana
depends_on:
- prometheus
- loki
# garage:
# image: dxflrs/garage:v1.0
# restart: unless-stopped
# volumes:
# - garage_data:/var/lib/garage
# - ./garage.toml:/etc/garage.toml:ro
networks:
# Default project network — kept implicit for production services.
# IPv6 is enabled so the journal can federate with v6-only instances
# (e.g. social.ullrich.is). Docker ≥27 auto-allocates a ULA subnet and
# NAT66-masquerades egress via the host's public v6 address.
# NOTE: flipping this on an existing deployment requires recreating the
# network: `docker compose down && docker compose --env-file .env up -d`.
default:
enable_ipv6: true
# Created by this compose project with a fixed (un-namespaced) name so the
# staging compose project can attach to it via `external: true`. Only
# postgres is joined here on the production side; staging containers join
# to reach postgres by hostname.
# IPv6-enabled so staging/preview journals — whose only network this is —
# get v6 federation egress like production. Recreating it on a live host
# requires detaching staging, previews, and postgres first.
trails-shared:
name: trails-shared
enable_ipv6: true
volumes:
caddy_data:
caddy_config:
pgdata:
prometheus_data:
loki_data:
grafana_data: