diff --git a/.github/workflows/cd-infra.yml b/.github/workflows/cd-infra.yml index 05f20e6..df41b93 100644 --- a/.github/workflows/cd-infra.yml +++ b/.github/workflows/cd-infra.yml @@ -43,7 +43,7 @@ jobs: host: ${{ secrets.DEPLOY_HOST }} username: root key: ${{ secrets.DEPLOY_SSH_KEY }} - source: "infrastructure/docker-compose.yml,infrastructure/caddy,infrastructure/prometheus/prometheus.yml,infrastructure/loki/loki-config.yml,infrastructure/promtail/promtail-config.yml,infrastructure/postgres/queries.yml,infrastructure/postgres/init-grafana-user.sql,infrastructure/grafana/provisioning,infrastructure/grafana/dashboards" + source: "infrastructure/docker-compose.yml,infrastructure/caddy,infrastructure/prometheus/prometheus.yml,infrastructure/loki/loki-config.yml,infrastructure/promtail/promtail-config.yml,infrastructure/postgres/queries.yml,infrastructure/postgres/init-grafana-user.sql,infrastructure/grafana/provisioning,infrastructure/grafana/dashboards,infrastructure/scripts" target: /opt/trails-cool strip_components: 1 diff --git a/e2e/fixtures/test.ts b/e2e/fixtures/test.ts index ea8c550..4af23cd 100644 --- a/e2e/fixtures/test.ts +++ b/e2e/fixtures/test.ts @@ -8,10 +8,11 @@ import { test as base, expect } from "@playwright/test"; * * The trigger for this safety net was #282 — a POI test that silently * relied on live Overpass data for months because its `page.route` - * was pointing at a URL the browser no longer hit after `/api/overpass` - * was introduced. With this fixture in place, a request to - * `overpass.private.coffee` (or any other unlisted host) would abort - * and the test would surface the missing mock immediately. + * was pointing at a URL the browser no longer hit after the POI proxy + * was introduced. With this fixture in place, a request to any unlisted + * host would abort and the test would surface the missing mock + * immediately. (POIs are now served from the same-origin `/api/pois` + * index endpoint, so the map makes no third-party POI requests at all.) */ const EXTERNAL_ALLOWLIST: RegExp[] = [ // App origins served by Playwright's webServer diff --git a/e2e/integration.test.ts b/e2e/integration.test.ts index 50c6811..059bcb7 100644 --- a/e2e/integration.test.ts +++ b/e2e/integration.test.ts @@ -328,14 +328,15 @@ test.describe("Integration: BRouter routing", () => { expect(response.status()).toBe(401); }); - test("rejects /api/overpass without an X-Trails-Session header", async ({ request }) => { - const response = await request.post(`${PLANNER}/api/overpass`, { - data: "data=[out:json];node[amenity=drinking_water](52.52,13.4,52.53,13.41);out;", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - Origin: PLANNER, + test("rejects /api/pois without an X-Trails-Session header", async ({ request }) => { + const response = await request.get( + `${PLANNER}/api/pois?bbox=52.52,13.40,52.53,13.41&categories=drinking_water`, + { + headers: { + Origin: PLANNER, + }, }, - }); + ); expect(response.status()).toBe(401); }); }); diff --git a/e2e/planner-overlays.test.ts b/e2e/planner-overlays.test.ts index d863a5f..3519cc2 100644 --- a/e2e/planner-overlays.test.ts +++ b/e2e/planner-overlays.test.ts @@ -30,17 +30,18 @@ test.describe("Planner – overlays", () => { test("enable POI category shows markers on map", async ({ page, request }) => { const url = await createSession(request); - await page.route("**/api/overpass", async (route) => { + await page.route("**/api/pois*", async (route) => { await route.fulfill({ status: 200, contentType: "application/json", body: JSON.stringify({ - elements: [ + pois: [ { - type: "node", id: 12345, lat: 52.52, lon: 13.405, + name: "Test Brunnen", + category: "drinking_water", tags: { amenity: "drinking_water", name: "Test Brunnen" }, }, ], diff --git a/e2e/planner-routing.test.ts b/e2e/planner-routing.test.ts index 2b0077e..e1d38af 100644 --- a/e2e/planner-routing.test.ts +++ b/e2e/planner-routing.test.ts @@ -108,17 +108,18 @@ test.describe("Planner – routing", () => { test("nearby POI snap moves waypoint and prepends note prefix", async ({ page, request }) => { const url = await createSession(request); - await page.route("**/api/overpass", async (route) => { + await page.route("**/api/pois*", async (route) => { await route.fulfill({ status: 200, contentType: "application/json", body: JSON.stringify({ - elements: [ + pois: [ { - type: "node", id: 99001, lat: 52.521, lon: 13.406, + name: "Stadtbrunnen", + category: "drinking_water", tags: { amenity: "drinking_water", name: "Stadtbrunnen" }, }, ], diff --git a/infrastructure/brouter-host/Caddyfile b/infrastructure/brouter-host/Caddyfile index b03a095..84536b7 100644 --- a/infrastructure/brouter-host/Caddyfile +++ b/infrastructure/brouter-host/Caddyfile @@ -24,11 +24,23 @@ @authed header X-BRouter-Auth {$BROUTER_AUTH_TOKEN} handle @authed { - reverse_proxy brouter:17777 { - # Don't forward the auth header upstream — BRouter doesn't - # use it, and it's cleaner to contain the credential at - # the proxy boundary. - header_up -X-BRouter-Auth + # POI index artifact + manifest published by poi-extract.sh. Served + # read-only from the mounted publish dir; the flagship import job + # pulls /poi/manifest.json and /poi/pois.ndjson.gz over the vSwitch. + # Same auth + vSwitch-only binding as BRouter, so it's unreachable + # from the public internet. + handle_path /poi/* { + root * /srv/poi + file_server + } + + handle { + reverse_proxy brouter:17777 { + # Don't forward the auth header upstream — BRouter doesn't + # use it, and it's cleaner to contain the credential at + # the proxy boundary. + header_up -X-BRouter-Auth + } } } diff --git a/infrastructure/brouter-host/docker-compose.yml b/infrastructure/brouter-host/docker-compose.yml index 4d82e4f..9958838 100644 --- a/infrastructure/brouter-host/docker-compose.yml +++ b/infrastructure/brouter-host/docker-compose.yml @@ -58,6 +58,9 @@ services: BROUTER_AUTH_TOKEN: ${BROUTER_AUTH_TOKEN:?BROUTER_AUTH_TOKEN must be set} volumes: - ./Caddyfile:/etc/caddy/Caddyfile:ro + # POI index artifact published by poi-extract.sh, served read-only under + # /poi/* on the same vSwitch-only listener as BRouter. + - ./poi-extract/publish:/srv/poi:ro - caddy-data:/data - caddy-config:/config networks: diff --git a/infrastructure/brouter-host/poi-extract/README.md b/infrastructure/brouter-host/poi-extract/README.md new file mode 100644 index 0000000..c59b307 --- /dev/null +++ b/infrastructure/brouter-host/poi-extract/README.md @@ -0,0 +1,92 @@ +# POI extract pipeline (BRouter host) + +Monthly job that turns an OSM PBF into the compact POI artifact the flagship's +`planner.pois` index is built from. This is the "filter where the disk is" +half of the [`poi-index`](../../../openspec/changes/poi-index/) change — it +replaces the third-party Overpass dependency for the Planner's POI overlays. + +Runs on the dedicated BRouter host (`ullrich.is`, the box with 1.8 TB free) +under the non-root `trails` user. The flagship import job +(`infrastructure/scripts/poi-import.sh`) pulls the result over the vSwitch. + +## Files + +| File | Role | +|------|------| +| `poi-extract.sh` | Download PBF → `osmium tags-filter` → `osmium export` → `to-ndjson.py` → gzip + manifest | +| `to-ndjson.py` | GeoJSONSeq → NDJSON, reducing ways/relations to their bbox centre (Overpass `out center` equivalent) | +| `osmium-filters.txt` | osmium tag filters, **generated** from `@trails-cool/map-core` POI selectors | +| `gen-osmium-filters.ts` | Regenerates `osmium-filters.txt` (run after changing categories) | +| `poi-extract.service` / `.timer` | systemd units (monthly) | + +## Single source of truth + +`osmium-filters.txt` is derived from `poiCategories` in +`packages/map-core/src/poi.ts`. After changing categories, regenerate it: + +```bash +npx tsx infrastructure/brouter-host/poi-extract/gen-osmium-filters.ts +``` + +`packages/map-core/src/osmium-filters.sync.test.ts` fails CI if the committed +file drifts from the selectors, so the host never needs a Node runtime. + +## Artifact contract + +`poi-extract.sh` writes to the publish dir (default `./publish`, mounted into +the Caddy sidecar at `/srv/poi`): + +- `pois.ndjson.gz` — one POI per line: + `{osm_type, osm_id, name, lat, lon, tags}`. Category is **not** stored here; + the flagship importer classifies each element from its `tags` using the same + map-core selectors (so the selector logic is never duplicated on this host). +- `manifest.json` — `{artifact, sha256, row_count, generated_at, source}`. + +Caddy serves both under `/poi/*` on the existing vSwitch-only `:17777` +listener, gated by the same `X-BRouter-Auth` shared secret as BRouter — so the +artifact is unreachable from the public internet. + +## Prerequisites + +Install on the host: `osmium-tool`, `python3`, `curl`, `gzip`, `coreutils` +(`sha256sum`). On Debian/Ubuntu: `apt install osmium-tool python3`. + +## Install the timer + +Deployed under `~trails/brouter/poi-extract/`. As a user unit with lingering: + +```bash +loginctl enable-linger trails +mkdir -p ~/.config/systemd/user +cp poi-extract.service poi-extract.timer ~/.config/systemd/user/ +systemctl --user daemon-reload +systemctl --user enable --now poi-extract.timer +systemctl --user list-timers poi-extract.timer +``` + +Trigger a run by hand (e.g. the first bootstrap): `systemctl --user start +poi-extract.service` then `journalctl --user -u poi-extract.service -f`. + +## Footprint + +- **Bandwidth**: the planet PBF is ~80 GB downloaded per run (monthly). Point + `POI_PBF_URL` at a Geofabrik mirror or a regional/continental extract to cut + this — e.g. `europe-latest.osm.pbf` (~30 GB) or `germany-latest.osm.pbf` + (~4 GB) for a smaller instance. +- **Disk**: the planet PBF + filtered PBF live transiently in `./work` and are + removed on exit (even on failure) via a trap. Peak transient usage ≈ size of + the PBF. The published artifact is a few hundred MB gzipped. +- **CPU/time**: a full planet filter is IO-bound and can take a few hours; the + unit runs at `Nice=10` / idle IO priority so it doesn't disturb BRouter. + +## Dry run first + +Before the first planet run, validate the whole chain on a small extract: + +```bash +POI_PBF_URL=https://download.geofabrik.de/europe/germany-latest.osm.pbf \ + ./poi-extract.sh +cat publish/manifest.json # row_count sane? sha256 present? +zcat publish/pois.ndjson.gz | head # records well-formed? +zcat publish/pois.ndjson.gz | wc -l +``` diff --git a/infrastructure/brouter-host/poi-extract/gen-osmium-filters.ts b/infrastructure/brouter-host/poi-extract/gen-osmium-filters.ts new file mode 100644 index 0000000..cd8e7d2 --- /dev/null +++ b/infrastructure/brouter-host/poi-extract/gen-osmium-filters.ts @@ -0,0 +1,18 @@ +/** + * Regenerate `osmium-filters.txt` from the POI category selectors in + * `@trails-cool/map-core` — the single source of truth for "what is a + * shelter". Run this after changing `poiCategories`: + * + * pnpm --filter @trails-cool/map-core exec tsx \ + * ../../infrastructure/brouter-host/poi-extract/gen-osmium-filters.ts + * + * The committed file is what the (Node-free) BRouter host reads at extract + * time. `osmium-filters.sync.test.ts` in map-core fails CI if the two drift. + */ +import { writeFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { osmiumTagFilters } from "../../../packages/map-core/src/poi.ts"; + +const out = fileURLToPath(new URL("./osmium-filters.txt", import.meta.url)); +writeFileSync(out, osmiumTagFilters().join("\n") + "\n"); +console.log(`Wrote ${out}`); diff --git a/infrastructure/brouter-host/poi-extract/osmium-filters.txt b/infrastructure/brouter-host/poi-extract/osmium-filters.txt new file mode 100644 index 0000000..e0d4f16 --- /dev/null +++ b/infrastructure/brouter-host/poi-extract/osmium-filters.txt @@ -0,0 +1,3 @@ +nwr/amenity=drinking_water,water_point,shelter,restaurant,cafe,fast_food,pub,biergarten,bicycle_parking,bicycle_repair_station,bicycle_rental,toilets +nwr/tourism=wilderness_hut,camp_site,caravan_site,hotel,hostel,guest_house,viewpoint +nwr/shop=supermarket,convenience,bakery diff --git a/infrastructure/brouter-host/poi-extract/poi-extract.service b/infrastructure/brouter-host/poi-extract/poi-extract.service new file mode 100644 index 0000000..5b751ce --- /dev/null +++ b/infrastructure/brouter-host/poi-extract/poi-extract.service @@ -0,0 +1,23 @@ +# POI extract — one-shot unit driven by poi-extract.timer. Runs as the +# non-root `trails` user (docker-group, scoped to ~trails/brouter/). Install as +# a user unit: `systemctl --user enable --now poi-extract.timer` with lingering +# enabled (`loginctl enable-linger trails`), or as a system unit with +# User=trails. See README.md. +[Unit] +Description=Filter an OSM PBF to trails.cool POI categories and publish the artifact +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +# Adjust WorkingDirectory to the deployed path (e.g. /home/trails/brouter/poi-extract). +WorkingDirectory=%h/brouter/poi-extract +ExecStart=%h/brouter/poi-extract/poi-extract.sh +# The planet download + filter is long-running and IO-heavy; give it room and +# keep it off the box's foreground priority. +Nice=10 +IOSchedulingClass=idle +TimeoutStartSec=6h + +[Install] +WantedBy=default.target diff --git a/infrastructure/brouter-host/poi-extract/poi-extract.sh b/infrastructure/brouter-host/poi-extract/poi-extract.sh new file mode 100755 index 0000000..5731b0c --- /dev/null +++ b/infrastructure/brouter-host/poi-extract/poi-extract.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# POI extract pipeline — runs on the dedicated BRouter host under the `trails` +# user (the box with 1.8 TB of disk). Downloads an OSM PBF, filters it to the +# planner's POI category selectors with osmium, reduces every element to a +# centroid, and publishes a compact NDJSON artifact + manifest that the +# flagship import job pulls over the vSwitch. +# +# Nothing here is planner/journal code — it's plain osmium + python so the host +# needs no Node runtime. The tag filter comes from `osmium-filters.txt`, which +# is generated from `@trails-cool/map-core`'s POI selectors (single source of +# truth; kept in sync by osmium-filters.sync.test.ts). +# +# Prerequisites on the host: osmium-tool, python3, curl, gzip, sha256sum. +# +# Usage: +# ./poi-extract.sh +# +# Env: +# POI_PBF_URL OSM PBF to filter. Default: the full planet. A self-hoster or +# a dry run can point this at a Geofabrik regional extract, e.g. +# https://download.geofabrik.de/europe/germany-latest.osm.pbf +# POI_WORK_DIR Scratch dir for the (large, transient) planet download. +# Default: