diff --git a/.github/workflows/cd-apps.yml b/.github/workflows/cd-apps.yml index 47d285d..212187e 100644 --- a/.github/workflows/cd-apps.yml +++ b/.github/workflows/cd-apps.yml @@ -100,7 +100,7 @@ jobs: host: ${{ secrets.DEPLOY_HOST }} username: root key: ${{ secrets.DEPLOY_SSH_KEY }} - source: "infrastructure/docker-compose.yml,infrastructure/Caddyfile" + source: "infrastructure/docker-compose.yml,infrastructure/caddy" target: /opt/trails-cool strip_components: 1 diff --git a/.github/workflows/cd-infra.yml b/.github/workflows/cd-infra.yml index 89c11bf..bdc3b31 100644 --- a/.github/workflows/cd-infra.yml +++ b/.github/workflows/cd-infra.yml @@ -43,7 +43,7 @@ jobs: host: ${{ secrets.DEPLOY_HOST }} username: root key: ${{ secrets.DEPLOY_SSH_KEY }} - source: "infrastructure/docker-compose.yml,infrastructure/Caddyfile,infrastructure/prometheus/prometheus.yml,infrastructure/loki/loki-config.yml,infrastructure/promtail/promtail-config.yml,infrastructure/postgres/queries.yml,infrastructure/postgres/init-grafana-user.sql,infrastructure/grafana/provisioning,infrastructure/grafana/dashboards" + source: "infrastructure/docker-compose.yml,infrastructure/caddy,infrastructure/prometheus/prometheus.yml,infrastructure/loki/loki-config.yml,infrastructure/promtail/promtail-config.yml,infrastructure/postgres/queries.yml,infrastructure/postgres/init-grafana-user.sql,infrastructure/grafana/provisioning,infrastructure/grafana/dashboards" target: /opt/trails-cool strip_components: 1 @@ -88,14 +88,25 @@ jobs: if [ "${{ github.event.inputs.restart_all }}" = "true" ]; then docker compose --env-file .env up -d --remove-orphans else - # Restart infra services (except Caddy — just reload its config). + # Restart infra services (config reloads handled below). # --remove-orphans cleans up containers whose service was deleted # from the compose file (e.g., the flagship `brouter` removal in # PR #297 left an orphan that had to be removed by hand). docker compose --env-file .env up -d --remove-orphans postgres prometheus loki promtail grafana postgres-exporter node-exporter cadvisor - docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile fi + # Apply config-only changes that `up -d` skips — it recreates a + # container only when its compose *definition* changes, not when a + # mounted config file's content changes. The configs are mounted as + # directories (not single files), so a reload/restart re-reads the + # freshly scp'd file; a single-file mount would have pinned the old + # inode. Prometheus hot-reloads on SIGHUP (zero downtime); Loki and + # Promtail reload their main config only on restart; Caddy reloads + # gracefully (validates, swaps live, no downtime). + docker compose --env-file .env kill -s SIGHUP prometheus + docker compose --env-file .env restart loki promtail + docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile + docker compose ps # Gate on the stack actually being up: postgres healthy and — diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index ff29112..dc489b3 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -36,7 +36,9 @@ services: ports: - "9090:9090" volumes: - - ./infrastructure/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + # Directory mount (not the file) to match prod — a single-file bind + # mount pins the host inode, so a replaced config is never seen. + - ./infrastructure/prometheus:/etc/prometheus:ro - prometheus_data:/prometheus grafana: @@ -60,9 +62,10 @@ services: ports: - "3100:3100" volumes: - - ./infrastructure/loki/loki-config.yml:/etc/loki/local-config.yaml:ro + # Directory mount (not the file) to match prod. + - ./infrastructure/loki:/etc/loki:ro - loki_data:/loki - command: ["-config.file=/etc/loki/local-config.yaml"] + command: ["-config.file=/etc/loki/loki-config.yml"] volumes: pgdata: diff --git a/infrastructure/Caddyfile b/infrastructure/caddy/Caddyfile similarity index 100% rename from infrastructure/Caddyfile rename to infrastructure/caddy/Caddyfile diff --git a/infrastructure/caddy/sites/.gitkeep b/infrastructure/caddy/sites/.gitkeep new file mode 100644 index 0000000..68f6879 --- /dev/null +++ b/infrastructure/caddy/sites/.gitkeep @@ -0,0 +1,3 @@ +# Mountpoint for the runtime ./sites overlay (per-PR staging snippets). +# Caddy reads site blocks from /etc/caddy/sites; this keeps the dir present +# inside the read-only ./caddy bind mount so the overlay can attach. diff --git a/infrastructure/docker-compose.yml b/infrastructure/docker-compose.yml index 57a8057..fb599e8 100644 --- a/infrastructure/docker-compose.yml +++ b/infrastructure/docker-compose.yml @@ -14,10 +14,17 @@ services: environment: DOMAIN: ${DOMAIN:-trails.cool} volumes: - - ./Caddyfile:/etc/caddy/Caddyfile:ro - # Per-PR staging snippets dropped in by cd-staging.yml. Bind mount - # auto-creates the host directory if it doesn't exist, so a fresh - # server doesn't need pre-provisioning. + # Mount the caddy/ DIRECTORY, not the Caddyfile directly. A single- + # file bind mount pins to the host file's inode at create time, so a + # deploy that replaces the file (scp) leaves the container reading the + # old inode — `caddy reload` then reloads stale config. A directory + # mount resolves children live, so reload picks up the new Caddyfile. + # The container path /etc/caddy/Caddyfile is unchanged. + - ./caddy:/etc/caddy:ro + # Per-PR staging snippets dropped in by cd-staging.yml, overlaid onto + # the read-only caddy dir above (caddy/sites/.gitkeep keeps the + # mountpoint present). Bind mount auto-creates the host directory if + # it doesn't exist, so a fresh server doesn't need pre-provisioning. - ./sites:/etc/caddy/sites:ro - caddy_data:/data - caddy_config:/config @@ -187,8 +194,10 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - /var/lib/docker/containers:/var/lib/docker/containers:ro - - ./promtail/promtail-config.yml:/etc/promtail/config.yml:ro - command: ["-config.file=/etc/promtail/config.yml"] + # Directory mount (not the file) so a deploy's replaced config is seen + # live and a SIGHUP reload picks it up — see the caddy note above. + - ./promtail:/etc/promtail:ro + command: ["-config.file=/etc/promtail/promtail-config.yml"] depends_on: - loki @@ -196,7 +205,10 @@ services: image: prom/prometheus:latest restart: unless-stopped volumes: - - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + # Directory mount (not the file) so config changes are seen live and a + # SIGHUP reload applies them without recreating the container — see the + # caddy note above. --config.file path below is unchanged. + - ./prometheus:/etc/prometheus:ro - prometheus_data:/prometheus command: - "--config.file=/etc/prometheus/prometheus.yml" @@ -207,9 +219,12 @@ services: image: grafana/loki:latest restart: unless-stopped volumes: - - ./loki/loki-config.yml:/etc/loki/local-config.yaml:ro + # Directory mount (not the file) so a replaced config is seen live — + # see the caddy note above. Loki only reloads its main config on + # restart (no SIGHUP), so cd-infra restarts loki rather than HUPs it. + - ./loki:/etc/loki:ro - loki_data:/loki - command: ["-config.file=/etc/loki/local-config.yaml"] + command: ["-config.file=/etc/loki/loki-config.yml"] # Publish only on the vSwitch IP so Promtail running on the # dedicated BRouter host can push logs in. Hetzner Cloud firewall # still blocks 3100 from the public internet. Internal services on