From b45e69885d3381f1d975f1787874529cca94c44e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= Date: Sun, 12 Jul 2026 22:38:03 +0200 Subject: [PATCH 01/11] poi-index: map-core selectors, planner.pois schema, /api/pois route + client, metrics Replace the planner's Overpass proxy with a self-hosted POI index: - map-core POI categories become structured tag selectors (single source of truth) + osmium filter / classification helpers - planner.pois PostGIS table (centroid points, GiST + category indexes) - /api/pois serving route (session + same-origin + rate limit, 100 cap) - lib/pois.ts client (renamed from overpass.ts; GET, quantized bbox) - metrics: poi_api_requests_total + DB-collected index rows/age gauges; drop overpass_* metrics - remove /api/overpass proxy + dead OVERPASS_URLS on the planner service (Journal surface backfill still uses it via code default) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../app/components/NearbyPoiMarkers.tsx | 2 +- apps/planner/app/components/PlannerMap.tsx | 2 +- .../app/components/WaypointSidebar.tsx | 2 +- apps/planner/app/lib/metrics.server.ts | 72 +++--- apps/planner/app/lib/overpass.server.ts | 111 --------- apps/planner/app/lib/overpass.test.ts | 199 ----------------- apps/planner/app/lib/overpass.ts | 211 ------------------ apps/planner/app/lib/poi-cache.test.ts | 2 +- apps/planner/app/lib/poi-cache.ts | 2 +- apps/planner/app/lib/poi-categories.test.ts | 2 +- apps/planner/app/lib/poi-snap.ts | 2 +- apps/planner/app/lib/pois.test.ts | 160 +++++++++++++ apps/planner/app/lib/pois.ts | 148 ++++++++++++ apps/planner/app/lib/snap-to-poi.test.ts | 2 +- apps/planner/app/lib/use-nearby-pois.ts | 4 +- apps/planner/app/lib/use-pois.ts | 4 +- apps/planner/app/routes.ts | 2 +- apps/planner/app/routes/api.overpass.test.ts | 209 ----------------- apps/planner/app/routes/api.overpass.ts | 127 ----------- apps/planner/app/routes/api.pois.test.ts | 159 +++++++++++++ apps/planner/app/routes/api.pois.ts | 140 ++++++++++++ infrastructure/.env.example | 8 +- infrastructure/docker-compose.staging.yml | 1 - infrastructure/docker-compose.yml | 6 - openspec/changes/poi-index/tasks.md | 20 +- packages/db/src/schema/planner.ts | 38 +++- packages/map-core/src/index.ts | 10 +- packages/map-core/src/poi.test.ts | 71 +++++- packages/map-core/src/poi.ts | 104 +++++++-- 29 files changed, 887 insertions(+), 933 deletions(-) delete mode 100644 apps/planner/app/lib/overpass.server.ts delete mode 100644 apps/planner/app/lib/overpass.test.ts delete mode 100644 apps/planner/app/lib/overpass.ts create mode 100644 apps/planner/app/lib/pois.test.ts create mode 100644 apps/planner/app/lib/pois.ts delete mode 100644 apps/planner/app/routes/api.overpass.test.ts delete mode 100644 apps/planner/app/routes/api.overpass.ts create mode 100644 apps/planner/app/routes/api.pois.test.ts create mode 100644 apps/planner/app/routes/api.pois.ts diff --git a/apps/planner/app/components/NearbyPoiMarkers.tsx b/apps/planner/app/components/NearbyPoiMarkers.tsx index d5878fd..0dc8167 100644 --- a/apps/planner/app/components/NearbyPoiMarkers.tsx +++ b/apps/planner/app/components/NearbyPoiMarkers.tsx @@ -1,6 +1,6 @@ import L from "leaflet"; import { Marker, Tooltip } from "react-leaflet"; -import type { Poi } from "~/lib/overpass"; +import type { Poi } from "~/lib/pois"; import type { PoiCategory } from "@trails-cool/map-core"; interface NearbyPoiMarkersProps { diff --git a/apps/planner/app/components/PlannerMap.tsx b/apps/planner/app/components/PlannerMap.tsx index e3a1284..2c1e9af 100644 --- a/apps/planner/app/components/PlannerMap.tsx +++ b/apps/planner/app/components/PlannerMap.tsx @@ -29,7 +29,7 @@ import { useGpxDrop } from "~/lib/use-gpx-drop"; import { useNearbyPois } from "~/lib/use-nearby-pois"; import { NearbyPoiMarkers } from "./NearbyPoiMarkers"; import { poiCategories } from "@trails-cool/map-core"; -import type { Poi } from "~/lib/overpass"; +import type { Poi } from "~/lib/pois"; import "leaflet/dist/leaflet.css"; function waypointIcon(index: number, overnight?: boolean, highlighted?: boolean, hasNote?: boolean): L.DivIcon { diff --git a/apps/planner/app/components/WaypointSidebar.tsx b/apps/planner/app/components/WaypointSidebar.tsx index 0c9c862..a0ed59b 100644 --- a/apps/planner/app/components/WaypointSidebar.tsx +++ b/apps/planner/app/components/WaypointSidebar.tsx @@ -6,7 +6,7 @@ import { setOvernight } from "~/lib/overnight"; import { DayBreakdown } from "./DayBreakdown"; import { useNearbyPois } from "~/lib/use-nearby-pois"; import { poiCategories } from "@trails-cool/map-core"; -import type { Poi } from "~/lib/overpass"; +import type { Poi } from "~/lib/pois"; import { extractWaypointData, waypointFromYMap, diff --git a/apps/planner/app/lib/metrics.server.ts b/apps/planner/app/lib/metrics.server.ts index 2aa8d9d..b025974 100644 --- a/apps/planner/app/lib/metrics.server.ts +++ b/apps/planner/app/lib/metrics.server.ts @@ -1,4 +1,7 @@ import client from "prom-client"; +import { sql } from "drizzle-orm"; +import { pois } from "@trails-cool/db/schema/planner"; +import { getDb } from "./db.ts"; // Guard all metric registration — Vite's dev server can re-evaluate // this module, causing "already registered" errors. @@ -41,39 +44,58 @@ export const brouterRequestDuration = getOrCreate("brouter_request_duration_seco }), ); -export const overpassCacheEvents = getOrCreate("overpass_cache_events_total", () => +// POI serving endpoint (`/api/pois`) request counter. `status` is the +// outcome class: ok | rate_limited | bad_request | forbidden | unauthorized +// | error. Replaces the former Overpass proxy/upstream metrics now that POIs +// are served from the local index. +export const poiApiRequests = getOrCreate("poi_api_requests_total", () => new client.Counter({ - name: "overpass_cache_events_total", - help: "Overpass proxy cache events", - labelNames: ["result"] as const, // hit | miss | coalesced + name: "poi_api_requests_total", + help: "POI serving endpoint requests by outcome status", + labelNames: ["status"] as const, }), ); -export const overpassCacheSize = getOrCreate("overpass_cache_size", () => +// Index size per category, computed from the planner's own DB at scrape time +// (prom-client binds `this` to the gauge inside `collect`). +export const poiIndexRows = getOrCreate("poi_index_rows", () => new client.Gauge({ - name: "overpass_cache_size", - help: "Current number of entries in the Overpass proxy cache", + name: "poi_index_rows", + help: "Number of POIs in the local index per category", + labelNames: ["category"] as const, + async collect() { + try { + const rows = await getDb() + .select({ category: pois.category, count: sql`count(*)::int` }) + .from(pois) + .groupBy(pois.category); + this.reset(); + for (const r of rows) this.set({ category: r.category }, r.count); + } catch { + // DB unavailable or table not yet created — leave last known values. + } + }, }), ); -export const overpassUpstreamDuration = getOrCreate("overpass_upstream_duration_seconds", () => - new client.Histogram({ - name: "overpass_upstream_duration_seconds", - help: "Duration of upstream Overpass API requests in seconds", - labelNames: ["upstream"] as const, - // Buckets go to 30s because our per-upstream timeout is 10s; a bit - // of headroom catches slow-but-not-timed-out tails without overly - // coarse resolution at the fast end where lz4 typically lands - // (~100–500ms). - buckets: [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 30], - }), -); - -export const overpassUpstreamRequests = getOrCreate("overpass_upstream_requests_total", () => - new client.Counter({ - name: "overpass_upstream_requests_total", - help: "Upstream Overpass API requests by upstream host and status", - labelNames: ["upstream", "status"] as const, +// Age of the freshest imported POI row, in seconds. Drives the stale-index +// alert (~6 weeks = one missed monthly refresh). +export const poiIndexAgeSeconds = getOrCreate("poi_index_age_seconds", () => + new client.Gauge({ + name: "poi_index_age_seconds", + help: "Seconds since the most recent POI index import", + async collect() { + try { + const [row] = await getDb() + .select({ + age: sql`extract(epoch from (now() - max(${pois.importedAt})))::float`, + }) + .from(pois); + if (row?.age != null) this.set(row.age); + } catch { + // DB unavailable or empty index — leave last known value. + } + }, }), ); diff --git a/apps/planner/app/lib/overpass.server.ts b/apps/planner/app/lib/overpass.server.ts deleted file mode 100644 index 6e78507..0000000 --- a/apps/planner/app/lib/overpass.server.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { - overpassUpstreamDuration, - overpassUpstreamRequests, -} from "~/lib/metrics.server"; - -/** - * Ordered list of upstream Overpass endpoints. We try each in turn - * until one returns a usable response; on timeout, network error, - * non-2xx status, or a body carrying Overpass's `rate_limited` runtime - * error, we fall over to the next. The default list keeps us on - * healthy community instances; `OVERPASS_URLS` overrides it and - * `OVERPASS_URL` is kept as a single-entry backward-compat alias. - */ -const DEFAULT_UPSTREAMS = [ - "https://lz4.overpass-api.de/api/interpreter", - "https://overpass-api.de/api/interpreter", -]; - -function loadUpstreams(): string[] { - const list = process.env.OVERPASS_URLS ?? process.env.OVERPASS_URL; - if (!list) return DEFAULT_UPSTREAMS; - return list.split(",").map((s) => s.trim()).filter(Boolean); -} - -export const UPSTREAMS = loadUpstreams(); -const USER_AGENT = "trails.cool Planner (https://trails.cool; legal@trails.cool)"; - -// Per-attempt wall-clock budget. Keep aggressive so failover kicks in -// quickly when an upstream is saturated — a single slow instance -// shouldn't cost the user minutes. -const PER_UPSTREAM_TIMEOUT_MS = 10_000; - -export interface UpstreamResult { - body: string; - contentType: string; - status: number; - cacheable: boolean; -} - -function hostOf(url: string): string { - try { - return new URL(url).hostname; - } catch { - return "unknown"; - } -} - -/** - * Try each upstream in order until one returns a usable response. - * Returns the first successful result, or `null` if every upstream - * failed. `clientSignal` is the browser's abort signal — if the user - * gives up before we succeed, we stop trying and propagate the abort - * to the in-flight upstream fetch so we don't waste its capacity on a - * request nobody wants anymore. - */ -export async function fetchWithFailover( - body: string, - clientSignal: AbortSignal, - urls: readonly string[] = UPSTREAMS, -): Promise { - for (const url of urls) { - if (clientSignal.aborted) break; - const upstream = hostOf(url); - const start = Date.now(); - try { - const signal = AbortSignal.any([ - clientSignal, - AbortSignal.timeout(PER_UPSTREAM_TIMEOUT_MS), - ]); - const response = await fetch(url, { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - "User-Agent": USER_AGENT, - }, - body, - signal, - }); - const responseBody = await response.text(); - overpassUpstreamDuration.observe({ upstream }, (Date.now() - start) / 1000); - overpassUpstreamRequests.inc({ upstream, status: String(response.status) }); - - // Overpass returns 200 with a `rate_limited` runtime error when - // the instance is throttling us. Treat that like a 5xx: try the - // next upstream. - if (!response.ok || responseBody.includes("rate_limited")) { - continue; - } - - return { - body: responseBody, - contentType: response.headers.get("content-type") ?? "application/json", - status: 200, - cacheable: true, - }; - } catch (err) { - overpassUpstreamDuration.observe({ upstream }, (Date.now() - start) / 1000); - // If the CLIENT aborted, stop the whole loop — the user doesn't - // want an answer anymore. Any other error (timeout, network, - // DNS) counts as "this upstream failed, try the next." - if (clientSignal.aborted) { - overpassUpstreamRequests.inc({ upstream, status: "client-abort" }); - throw err; - } - const label = (err as Error).name === "TimeoutError" ? "timeout" : "error"; - overpassUpstreamRequests.inc({ upstream, status: label }); - continue; - } - } - return null; -} diff --git a/apps/planner/app/lib/overpass.test.ts b/apps/planner/app/lib/overpass.test.ts deleted file mode 100644 index a6036ae..0000000 --- a/apps/planner/app/lib/overpass.test.ts +++ /dev/null @@ -1,199 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { buildQuery, parseResponse, deduplicateById, quantizeBbox, fetchNearbyPois, type Poi } from "./overpass.ts"; -import { poiCategories } from "@trails-cool/map-core"; - -describe("buildQuery", () => { - it("builds Overpass QL with bbox and category union", () => { - const bbox = { south: 52.5, west: 13.3, north: 52.6, east: 13.5 }; - const categories = poiCategories.filter((c) => c.id === "drinking_water"); - const query = buildQuery(bbox, categories); - - expect(query).toContain("[out:json]"); - expect(query).toContain("[bbox:52.500,13.300,52.600,13.500]"); - expect(query).toContain('amenity"="drinking_water"'); - expect(query).toContain("out center qt 100"); - }); - - it("combines multiple categories into a single union", () => { - const bbox = { south: 52.5, west: 13.3, north: 52.6, east: 13.5 }; - const categories = poiCategories.filter((c) => ["drinking_water", "camping"].includes(c.id)); - const query = buildQuery(bbox, categories); - - expect(query).toContain("drinking_water"); - expect(query).toContain("camp_site"); - }); - - it("produces identical queries for near-identical viewports within one grid cell", () => { - const categories = poiCategories.filter((c) => c.id === "drinking_water"); - const a = buildQuery( - { south: 52.5041, west: 13.3072, north: 52.5931, east: 13.4928 }, - categories, - ); - const b = buildQuery( - { south: 52.5039, west: 13.3078, north: 52.5925, east: 13.4921 }, - categories, - ); - expect(a).toBe(b); - }); - - it("expands the bbox outward so the original viewport is always covered", () => { - const categories = poiCategories.filter((c) => c.id === "drinking_water"); - const query = buildQuery( - { south: 52.5041, west: 13.3072, north: 52.5931, east: 13.4928 }, - categories, - ); - expect(query).toContain("[bbox:52.500,13.300,52.600,13.500]"); - }); -}); - -describe("quantizeBbox", () => { - it("snaps south and west down, north and east up", () => { - const q = quantizeBbox({ south: 52.5041, west: 13.3072, north: 52.5931, east: 13.4928 }, 0.01); - expect(q.south).toBeCloseTo(52.5, 10); - expect(q.west).toBeCloseTo(13.3, 10); - expect(q.north).toBeCloseTo(52.6, 10); - expect(q.east).toBeCloseTo(13.5, 10); - }); - - it("is idempotent on already-aligned coordinates", () => { - const bbox = { south: 52.5, west: 13.3, north: 52.6, east: 13.5 }; - const q = quantizeBbox(bbox, 0.01); - expect(q.south).toBeCloseTo(52.5, 10); - expect(q.north).toBeCloseTo(52.6, 10); - }); -}); - -describe("parseResponse", () => { - it("parses nodes into Poi objects", () => { - const data = { - elements: [ - { - type: "node", - id: 123, - lat: 52.52, - lon: 13.405, - tags: { amenity: "drinking_water", name: "Brunnen" }, - }, - ], - }; - const categories = poiCategories.filter((c) => c.id === "drinking_water"); - const pois = parseResponse(data, categories); - - expect(pois).toHaveLength(1); - expect(pois[0]!.id).toBe(123); - expect(pois[0]!.name).toBe("Brunnen"); - expect(pois[0]!.category).toBe("drinking_water"); - expect(pois[0]!.lat).toBe(52.52); - }); - - it("uses center for way/relation elements", () => { - const data = { - elements: [ - { - type: "way", - id: 456, - center: { lat: 52.51, lon: 13.39 }, - tags: { tourism: "camp_site", name: "Zeltplatz" }, - }, - ], - }; - const categories = poiCategories.filter((c) => c.id === "camping"); - const pois = parseResponse(data, categories); - - expect(pois).toHaveLength(1); - expect(pois[0]!.lat).toBe(52.51); - expect(pois[0]!.lon).toBe(13.39); - }); - - it("skips elements without coordinates", () => { - const data = { - elements: [ - { type: "node", id: 789, tags: { amenity: "drinking_water" } }, - ], - }; - const categories = poiCategories.filter((c) => c.id === "drinking_water"); - const pois = parseResponse(data, categories); - expect(pois).toHaveLength(0); - }); - - it("skips elements that don't match any category", () => { - const data = { - elements: [ - { type: "node", id: 100, lat: 52.52, lon: 13.4, tags: { amenity: "bank" } }, - ], - }; - const categories = poiCategories.filter((c) => c.id === "drinking_water"); - const pois = parseResponse(data, categories); - expect(pois).toHaveLength(0); - }); -}); - -describe("deduplicateById", () => { - it("removes duplicate POIs by id", () => { - const pois: Poi[] = [ - { id: 1, lat: 52.5, lon: 13.4, category: "drinking_water", tags: {} }, - { id: 1, lat: 52.5, lon: 13.4, category: "camping", tags: {} }, - { id: 2, lat: 52.6, lon: 13.5, category: "shelter", tags: {} }, - ]; - const result = deduplicateById(pois); - expect(result).toHaveLength(2); - expect(result[0]!.id).toBe(1); - expect(result[1]!.id).toBe(2); - }); -}); - -describe("fetchNearbyPois", () => { - const emptyOverpassResponse = { elements: [] }; - - beforeEach(() => { - vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ - ok: true, - text: async () => JSON.stringify(emptyOverpassResponse), - json: async () => emptyOverpassResponse, - })); - }); - - afterEach(() => { - vi.unstubAllGlobals(); - }); - - it("constructs a bbox from lat/lon/radius and calls fetch", async () => { - const categories = poiCategories.filter((c) => c.id === "drinking_water"); - await fetchNearbyPois(50.0, 10.0, 500, categories); - - expect(fetch).toHaveBeenCalledOnce(); - const rawBody = (((fetch as ReturnType).mock.calls[0] as unknown[])[1]) as unknown as { body: string }; - const body = decodeURIComponent(rawBody.body.replace("data=", "")); - // bbox quantized to 0.01° grid; 500m at lat=50 well within one cell - expect(body).toMatch(/\[bbox:4\d\.\d+,\d+\.\d+,50\.\d+,10\.\d+\]/); - }); - - it("bbox is symmetric around the input point", async () => { - const lat = 48.0; - const lon = 11.0; - const radius = 1000; - const categories = poiCategories.filter((c) => c.id === "drinking_water"); - await fetchNearbyPois(lat, lon, radius, categories); - - const rawBody = (((fetch as ReturnType).mock.calls[0] as unknown[])[1]) as unknown as { body: string }; - const body = decodeURIComponent(rawBody.body.replace("data=", "")); - const match = body.match(/\[bbox:([\d.]+),([\d.]+),([\d.]+),([\d.]+)\]/); - expect(match).not.toBeNull(); - const [, south, west, north, east] = (match as RegExpMatchArray).map(Number); - // After quantization the center might not be exactly lat/lon, but the extent should be ≥ radius - const DEG_PER_METER_LAT = 1 / 111320; - const dLat = radius * DEG_PER_METER_LAT; - expect((north as number) - (south as number)).toBeGreaterThanOrEqual(dLat * 2 - 0.02); - expect((east as number) - (west as number)).toBeGreaterThan(0); - }); - - it("forwards AbortSignal to fetch", async () => { - const controller = new AbortController(); - const categories = poiCategories.filter((c) => c.id === "drinking_water"); - await fetchNearbyPois(50.0, 10.0, 500, categories, controller.signal); - - expect(fetch).toHaveBeenCalledOnce(); - const callOptions = (((fetch as ReturnType).mock.calls[0] as unknown[])[1]) as unknown as { signal: AbortSignal }; - expect(callOptions?.signal).toBe(controller.signal); - }); -}); diff --git a/apps/planner/app/lib/overpass.ts b/apps/planner/app/lib/overpass.ts deleted file mode 100644 index cccc642..0000000 --- a/apps/planner/app/lib/overpass.ts +++ /dev/null @@ -1,211 +0,0 @@ -import type { PoiCategory } from "@trails-cool/map-core"; - -const OVERPASS_PROXY = "/api/overpass"; - -// Snap bbox coordinates to this grid before building the query so that two -// users looking at nearly-identical viewports produce byte-identical queries -// and therefore share a server-side cache entry. Expansion is outward -// (south/west floor, north/east ceil) so the user's viewport is always -// covered. ~0.01° ≈ 1 km at mid-latitudes. -const BBOX_GRID_STEP = 0.01; - -// Decimals used when formatting the quantized bbox into the query string. -// 3 decimals → 0.001° precision (~111 m) which is well below BBOX_GRID_STEP, -// so the string is a stable representation of the quantized cell. -const BBOX_DECIMALS = 3; - -export interface Poi { - id: number; - lat: number; - lon: number; - name?: string; - category: string; - tags: Record; -} - -export interface BBox { - south: number; - west: number; - north: number; - east: number; -} - -/** - * Quantize a bbox to the grid defined by `BBOX_GRID_STEP`, expanding outward - * so the original bbox is fully contained. Returns a new BBox whose - * coordinates are cell-aligned. - */ -export function quantizeBbox(bbox: BBox, step: number = BBOX_GRID_STEP): BBox { - return { - south: Math.floor(bbox.south / step) * step, - west: Math.floor(bbox.west / step) * step, - north: Math.ceil(bbox.north / step) * step, - east: Math.ceil(bbox.east / step) * step, - }; -} - -/** - * Build an Overpass QL query combining all enabled categories into a union. - * The bbox is quantized to a fixed grid and formatted with a fixed number of - * decimals so near-identical viewports produce a byte-identical query — which - * the `/api/overpass` server-side cache keys on. - */ -export function buildQuery(bbox: BBox, categories: PoiCategory[]): string { - const q = quantizeBbox(bbox); - const bboxStr = [q.south, q.west, q.north, q.east] - .map((n) => n.toFixed(BBOX_DECIMALS)) - .join(","); - const unions = categories.map((c) => c.query).join(""); - return `[out:json][timeout:10][maxsize:1048576][bbox:${bboxStr}];(${unions});out center qt 100;`; -} - -/** - * Parse Overpass JSON response into typed Poi objects. - */ -export function parseResponse( - data: { elements: Array<{ - type: string; - id: number; - lat?: number; - lon?: number; - center?: { lat: number; lon: number }; - tags?: Record; - }> }, - categories: PoiCategory[], -): Poi[] { - const pois: Poi[] = []; - - for (const el of data.elements) { - const lat = el.lat ?? el.center?.lat; - const lon = el.lon ?? el.center?.lon; - if (lat === undefined || lon === undefined) continue; - - const tags = el.tags ?? {}; - const category = matchCategory(tags, categories); - if (!category) continue; - - pois.push({ - id: el.id, - lat, - lon, - name: tags.name, - category: category.id, - tags, - }); - } - - return deduplicateById(pois); -} - -/** - * Match an element's tags to the first matching category. - */ -function matchCategory(tags: Record, categories: PoiCategory[]): PoiCategory | null { - for (const cat of categories) { - // Parse query fragments like 'nwr["amenity"="drinking_water"];' - const fragments = cat.query.split(";").filter(Boolean); - for (const frag of fragments) { - const match = frag.match(/\["(\w+)"="([^"]+)"\]/); - if (match && tags[match[1]!] === match[2]) return cat; - } - } - return null; -} - -/** - * Deduplicate POIs by OSM node ID (same node may match multiple queries). - */ -export function deduplicateById(pois: Poi[]): Poi[] { - const seen = new Set(); - return pois.filter((poi) => { - if (seen.has(poi.id)) return false; - seen.add(poi.id); - return true; - }); -} - -/** - * Query the Overpass API for POIs within a bounding box. - * - * All queries route through the Planner's own `/api/overpass` proxy, which - * adds a User-Agent identifying trails.cool, rate-limits per IP, and enforces - * same-origin. The client never talks to a public Overpass host directly. - */ -export async function queryPois( - bbox: BBox, - categories: PoiCategory[], - sessionId: string, - signal?: AbortSignal, -): Promise { - if (categories.length === 0) return []; - - const query = buildQuery(bbox, categories); - - const response = await fetch(OVERPASS_PROXY, { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - // Bind this call to the active planner session so the proxy - // isn't anonymously reachable. - "X-Trails-Session": sessionId, - }, - body: `data=${encodeURIComponent(query)}`, - signal, - }); - - if (response.status === 429) { - throw new OverpassRateLimitError(); - } - - if (!response.ok) { - throw new Error(`Overpass API error: ${response.status}`); - } - - const text = await response.text(); - - if (text.includes("rate_limited")) { - throw new OverpassRateLimitError(); - } - - let data; - try { - data = JSON.parse(text); - } catch { - throw new Error("Overpass API returned invalid JSON"); - } - - return parseResponse(data, categories); -} - -export class OverpassRateLimitError extends Error { - constructor() { - super("Overpass API rate limit exceeded"); - this.name = "OverpassRateLimitError"; - } -} - -// Degrees of latitude per meter (approximate, valid globally). -const DEG_PER_METER_LAT = 1 / 111320; - -/** - * Fetch POIs within `radiusMeters` of a coordinate. - * Builds a bbox around the point and reuses queryPois + the existing proxy. - */ -export async function fetchNearbyPois( - lat: number, - lon: number, - radiusMeters: number, - categories: import("@trails-cool/map-core").PoiCategory[], - signal?: AbortSignal, - sessionId = "nearby", -): Promise { - const dLat = radiusMeters * DEG_PER_METER_LAT; - const dLon = dLat / Math.cos((lat * Math.PI) / 180); - const bbox: BBox = { - south: lat - dLat, - north: lat + dLat, - west: lon - dLon, - east: lon + dLon, - }; - return queryPois(bbox, categories, sessionId, signal); -} diff --git a/apps/planner/app/lib/poi-cache.test.ts b/apps/planner/app/lib/poi-cache.test.ts index 1d20c98..c9f5459 100644 --- a/apps/planner/app/lib/poi-cache.test.ts +++ b/apps/planner/app/lib/poi-cache.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, beforeEach } from "vitest"; import { quantize, getCellKeys, getCached, setCached, clearCache } from "./poi-cache.ts"; -import type { Poi } from "./overpass.ts"; +import type { Poi } from "./pois.ts"; beforeEach(() => { clearCache(); diff --git a/apps/planner/app/lib/poi-cache.ts b/apps/planner/app/lib/poi-cache.ts index 2ee7eb2..4f8afeb 100644 --- a/apps/planner/app/lib/poi-cache.ts +++ b/apps/planner/app/lib/poi-cache.ts @@ -1,4 +1,4 @@ -import type { Poi, BBox } from "./overpass.ts"; +import type { Poi, BBox } from "./pois.ts"; const CELL_SIZE = 0.1; // degrees const TTL = 10 * 60 * 1000; // 10 minutes diff --git a/apps/planner/app/lib/poi-categories.test.ts b/apps/planner/app/lib/poi-categories.test.ts index a415aaa..f07983a 100644 --- a/apps/planner/app/lib/poi-categories.test.ts +++ b/apps/planner/app/lib/poi-categories.test.ts @@ -40,7 +40,7 @@ describe("poiCategories", () => { expect(cat.name).toBeTruthy(); expect(cat.icon).toBeTruthy(); expect(cat.color).toMatch(/^#/); - expect(cat.query).toContain("nwr"); + expect(cat.selectors.length).toBeGreaterThan(0); } }); }); diff --git a/apps/planner/app/lib/poi-snap.ts b/apps/planner/app/lib/poi-snap.ts index c35ea59..bb11332 100644 --- a/apps/planner/app/lib/poi-snap.ts +++ b/apps/planner/app/lib/poi-snap.ts @@ -1,4 +1,4 @@ -import type { Poi } from "./overpass.ts"; +import type { Poi } from "./pois.ts"; import { SNAP_DISTANCE_METERS } from "@trails-cool/map-core"; export interface SnapResult { diff --git a/apps/planner/app/lib/pois.test.ts b/apps/planner/app/lib/pois.test.ts new file mode 100644 index 0000000..cba7905 --- /dev/null +++ b/apps/planner/app/lib/pois.test.ts @@ -0,0 +1,160 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { + buildPoisQuery, + quantizeBbox, + deduplicateById, + queryPois, + fetchNearbyPois, + PoiRateLimitError, + type Poi, +} from "./pois.ts"; +import { poiCategories } from "@trails-cool/map-core"; + +const drinkingWater = poiCategories.filter((c) => c.id === "drinking_water"); + +describe("buildPoisQuery", () => { + it("encodes the quantized bbox and category ids", () => { + const bbox = { south: 52.5, west: 13.3, north: 52.6, east: 13.5 }; + const q = buildPoisQuery(bbox, drinkingWater); + expect(q).toContain("bbox=52.500,13.300,52.600,13.500"); + expect(q).toContain("categories=drinking_water"); + }); + + it("joins multiple categories", () => { + const cats = poiCategories.filter((c) => ["drinking_water", "camping"].includes(c.id)); + const q = buildPoisQuery({ south: 52.5, west: 13.3, north: 52.6, east: 13.5 }, cats); + expect(decodeURIComponent(q)).toContain("categories=drinking_water,camping"); + }); + + it("produces identical query strings for near-identical viewports in one cell", () => { + const a = buildPoisQuery( + { south: 52.5041, west: 13.3072, north: 52.5931, east: 13.4928 }, + drinkingWater, + ); + const b = buildPoisQuery( + { south: 52.5039, west: 13.3078, north: 52.5925, east: 13.4921 }, + drinkingWater, + ); + expect(a).toBe(b); + }); +}); + +describe("quantizeBbox", () => { + it("snaps south/west down and north/east up", () => { + const q = quantizeBbox({ south: 52.5041, west: 13.3072, north: 52.5931, east: 13.4928 }, 0.01); + expect(q.south).toBeCloseTo(52.5, 10); + expect(q.west).toBeCloseTo(13.3, 10); + expect(q.north).toBeCloseTo(52.6, 10); + expect(q.east).toBeCloseTo(13.5, 10); + }); +}); + +describe("deduplicateById", () => { + it("keeps the first occurrence of each id", () => { + const pois: Poi[] = [ + { id: 1, lat: 52.5, lon: 13.4, category: "drinking_water", tags: {} }, + { id: 1, lat: 52.5, lon: 13.4, category: "camping", tags: {} }, + { id: 2, lat: 52.6, lon: 13.5, category: "shelter", tags: {} }, + ]; + const result = deduplicateById(pois); + expect(result.map((p) => p.id)).toEqual([1, 2]); + }); +}); + +describe("queryPois", () => { + afterEach(() => vi.unstubAllGlobals()); + + it("GETs /api/pois with the session header and parses the response", async () => { + const fetchSpy = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ + pois: [{ id: 7, lat: 52.5, lon: 13.4, name: "Brunnen", category: "drinking_water", tags: {} }], + }), + }); + vi.stubGlobal("fetch", fetchSpy); + + const pois = await queryPois( + { south: 52.5, west: 13.3, north: 52.6, east: 13.5 }, + drinkingWater, + "sess-1", + ); + + expect(pois).toHaveLength(1); + expect(pois[0]!.id).toBe(7); + const [url, opts] = fetchSpy.mock.calls[0] as [string, RequestInit]; + expect(url).toContain("/api/pois?bbox="); + expect(url).toContain("categories=drinking_water"); + expect((opts.headers as Record)["X-Trails-Session"]).toBe("sess-1"); + expect(opts.method).toBe("GET"); + }); + + it("returns [] without fetching when no categories are enabled", async () => { + const fetchSpy = vi.fn(); + vi.stubGlobal("fetch", fetchSpy); + const pois = await queryPois({ south: 52.5, west: 13.3, north: 52.6, east: 13.5 }, [], "s"); + expect(pois).toEqual([]); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it("throws PoiRateLimitError on a 429", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: false, status: 429 })); + await expect( + queryPois({ south: 52.5, west: 13.3, north: 52.6, east: 13.5 }, drinkingWater, "s"), + ).rejects.toBeInstanceOf(PoiRateLimitError); + }); + + it("throws a generic error on other non-ok statuses (e.g. 503)", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: false, status: 503 })); + await expect( + queryPois({ south: 52.5, west: 13.3, north: 52.6, east: 13.5 }, drinkingWater, "s"), + ).rejects.toThrow("POI service error: 503"); + }); + + it("deduplicates POIs returned across categories", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ + pois: [ + { id: 1, lat: 52.5, lon: 13.4, category: "drinking_water", tags: {} }, + { id: 1, lat: 52.5, lon: 13.4, category: "toilets", tags: {} }, + ], + }), + }), + ); + const pois = await queryPois( + { south: 52.5, west: 13.3, north: 52.6, east: 13.5 }, + drinkingWater, + "s", + ); + expect(pois).toHaveLength(1); + }); +}); + +describe("fetchNearbyPois", () => { + beforeEach(() => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue({ ok: true, status: 200, json: async () => ({ pois: [] }) }), + ); + }); + afterEach(() => vi.unstubAllGlobals()); + + it("builds a bbox around the point and calls /api/pois", async () => { + await fetchNearbyPois(50.0, 10.0, 500, drinkingWater); + expect(fetch).toHaveBeenCalledOnce(); + const [url] = (fetch as ReturnType).mock.calls[0] as [string]; + expect(url).toMatch(/\/api\/pois\?bbox=\d+\.\d+,\d+\.\d+,\d+\.\d+,\d+\.\d+/); + expect(url).toContain("categories=drinking_water"); + }); + + it("forwards the AbortSignal to fetch", async () => { + const controller = new AbortController(); + await fetchNearbyPois(50.0, 10.0, 500, drinkingWater, controller.signal); + const [, opts] = (fetch as ReturnType).mock.calls[0] as [string, RequestInit]; + expect(opts.signal).toBe(controller.signal); + }); +}); diff --git a/apps/planner/app/lib/pois.ts b/apps/planner/app/lib/pois.ts new file mode 100644 index 0000000..01898e8 --- /dev/null +++ b/apps/planner/app/lib/pois.ts @@ -0,0 +1,148 @@ +import type { PoiCategory } from "@trails-cool/map-core"; + +const POIS_ENDPOINT = "/api/pois"; + +// Snap bbox coordinates to this grid before requesting so that two users +// looking at nearly-identical viewports produce the same URL and can share a +// browser-cache entry. Expansion is outward (south/west floor, north/east +// ceil) so the user's viewport is always covered. ~0.01° ≈ 1 km at +// mid-latitudes. +const BBOX_GRID_STEP = 0.01; + +// Decimals used when formatting the quantized bbox into the request URL. +// 3 decimals → 0.001° precision (~111 m), well below BBOX_GRID_STEP, so the +// string is a stable representation of the quantized cell. +const BBOX_DECIMALS = 3; + +export interface Poi { + id: number; + lat: number; + lon: number; + name?: string; + category: string; + tags: Record; +} + +export interface BBox { + south: number; + west: number; + north: number; + east: number; +} + +/** + * Quantize a bbox to the grid defined by `BBOX_GRID_STEP`, expanding outward + * so the original bbox is fully contained. Returns a new BBox whose + * coordinates are cell-aligned. + */ +export function quantizeBbox(bbox: BBox, step: number = BBOX_GRID_STEP): BBox { + return { + south: Math.floor(bbox.south / step) * step, + west: Math.floor(bbox.west / step) * step, + north: Math.ceil(bbox.north / step) * step, + east: Math.ceil(bbox.east / step) * step, + }; +} + +/** + * Build the `/api/pois` query string for a bbox + categories. The bbox is + * quantized and formatted with fixed decimals so near-identical viewports + * produce a byte-identical URL — which the browser cache keys on. + */ +export function buildPoisQuery(bbox: BBox, categories: PoiCategory[]): string { + const q = quantizeBbox(bbox); + const bboxStr = [q.south, q.west, q.north, q.east] + .map((n) => n.toFixed(BBOX_DECIMALS)) + .join(","); + const cats = categories.map((c) => c.id).join(","); + return `bbox=${bboxStr}&categories=${encodeURIComponent(cats)}`; +} + +/** + * Deduplicate POIs by OSM id (an element in two enabled categories comes back + * once per category; the map shows a single marker for it). + */ +export function deduplicateById(pois: Poi[]): Poi[] { + const seen = new Set(); + return pois.filter((poi) => { + if (seen.has(poi.id)) return false; + seen.add(poi.id); + return true; + }); +} + +/** + * Query the instance's own POI index for POIs within a bounding box. + * + * All requests go to the Planner's `/api/pois` endpoint, which enforces + * same-origin + session binding and a per-IP rate limit, and serves from the + * local PostGIS index. No POI request ever leaves trails.cool infrastructure. + */ +export async function queryPois( + bbox: BBox, + categories: PoiCategory[], + sessionId: string, + signal?: AbortSignal, +): Promise { + if (categories.length === 0) return []; + + const response = await fetch(`${POIS_ENDPOINT}?${buildPoisQuery(bbox, categories)}`, { + method: "GET", + headers: { + // Bind this call to the active planner session so the endpoint isn't + // anonymously reachable. + "X-Trails-Session": sessionId, + }, + signal, + }); + + if (response.status === 429) { + throw new PoiRateLimitError(); + } + + if (!response.ok) { + throw new Error(`POI service error: ${response.status}`); + } + + let data: { pois?: Poi[] }; + try { + data = await response.json(); + } catch { + throw new Error("POI service returned invalid JSON"); + } + + return deduplicateById(data.pois ?? []); +} + +export class PoiRateLimitError extends Error { + constructor() { + super("POI service rate limit exceeded"); + this.name = "PoiRateLimitError"; + } +} + +// Degrees of latitude per meter (approximate, valid globally). +const DEG_PER_METER_LAT = 1 / 111320; + +/** + * Fetch POIs within `radiusMeters` of a coordinate. + * Builds a bbox around the point and reuses queryPois + the `/api/pois` route. + */ +export async function fetchNearbyPois( + lat: number, + lon: number, + radiusMeters: number, + categories: PoiCategory[], + signal?: AbortSignal, + sessionId = "nearby", +): Promise { + const dLat = radiusMeters * DEG_PER_METER_LAT; + const dLon = dLat / Math.cos((lat * Math.PI) / 180); + const bbox: BBox = { + south: lat - dLat, + north: lat + dLat, + west: lon - dLon, + east: lon + dLon, + }; + return queryPois(bbox, categories, sessionId, signal); +} diff --git a/apps/planner/app/lib/snap-to-poi.test.ts b/apps/planner/app/lib/snap-to-poi.test.ts index 70a6f3a..3754626 100644 --- a/apps/planner/app/lib/snap-to-poi.test.ts +++ b/apps/planner/app/lib/snap-to-poi.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect } from "vitest"; import * as Y from "yjs"; import { poiCategories } from "@trails-cool/map-core"; -import type { Poi } from "./overpass.ts"; +import type { Poi } from "./pois.ts"; // Mirrors the snapToPoi logic from WaypointSidebar / PlannerMap function snapToPoi( diff --git a/apps/planner/app/lib/use-nearby-pois.ts b/apps/planner/app/lib/use-nearby-pois.ts index 3b6155e..0b08ed8 100644 --- a/apps/planner/app/lib/use-nearby-pois.ts +++ b/apps/planner/app/lib/use-nearby-pois.ts @@ -1,5 +1,5 @@ import { useState, useEffect, useRef } from "react"; -import { fetchNearbyPois, OverpassRateLimitError, type Poi } from "./overpass.ts"; +import { fetchNearbyPois, PoiRateLimitError, type Poi } from "./pois.ts"; import { poiCategories } from "@trails-cool/map-core"; const NEARBY_RADIUS_METERS = 500; @@ -53,7 +53,7 @@ export function useNearbyPois( } } catch (err) { if (controller.signal.aborted) return; - if (err instanceof OverpassRateLimitError) { + if (err instanceof PoiRateLimitError) { rateLimitedUntilRef.current = Date.now() + RATE_LIMIT_SUPPRESS_MS; setState({ pois: [], status: "rate_limited" }); } else { diff --git a/apps/planner/app/lib/use-pois.ts b/apps/planner/app/lib/use-pois.ts index b016a0a..0091355 100644 --- a/apps/planner/app/lib/use-pois.ts +++ b/apps/planner/app/lib/use-pois.ts @@ -1,5 +1,5 @@ import { useCallback, useEffect, useRef, useState } from "react"; -import { queryPois, OverpassRateLimitError, type Poi, type BBox } from "./overpass.ts"; +import { queryPois, PoiRateLimitError, type Poi, type BBox } from "./pois.ts"; import { getCached, setCached } from "./poi-cache.ts"; import { poiCategories } from "@trails-cool/map-core"; @@ -86,7 +86,7 @@ export function usePois(sessionId: string): PoiState { } catch (err) { if (controller.signal.aborted) return; - if (err instanceof OverpassRateLimitError) { + if (err instanceof PoiRateLimitError) { setStatus("rate_limited"); backoffRef.current = Math.min( (backoffRef.current || BACKOFF_BASE_MS) * 2, diff --git a/apps/planner/app/routes.ts b/apps/planner/app/routes.ts index ee58808..53f07f0 100644 --- a/apps/planner/app/routes.ts +++ b/apps/planner/app/routes.ts @@ -6,7 +6,7 @@ export default [ route("api/sessions", "routes/api.sessions.ts"), route("api/route", "routes/api.route.ts"), route("api/route-segments", "routes/api.route-segments.ts"), - route("api/overpass", "routes/api.overpass.ts"), + route("api/pois", "routes/api.pois.ts"), route("api/save-to-journal", "routes/api.save-to-journal.ts"), route("session/:id", "routes/session.$id.tsx"), ] satisfies RouteConfig; diff --git a/apps/planner/app/routes/api.overpass.test.ts b/apps/planner/app/routes/api.overpass.test.ts deleted file mode 100644 index 65212b5..0000000 --- a/apps/planner/app/routes/api.overpass.test.ts +++ /dev/null @@ -1,209 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; - -// Metrics module pulls in prom-client and registers processes-wide -// metrics on import; mock it out so tests aren't entangled with -// the registry and can assert on label values instead. -vi.mock("~/lib/metrics.server", () => ({ - overpassCacheEvents: { inc: vi.fn() }, - overpassCacheSize: { set: vi.fn() }, - overpassUpstreamDuration: { observe: vi.fn() }, - overpassUpstreamRequests: { inc: vi.fn() }, -})); - -import { fetchWithFailover } from "~/lib/overpass.server"; -import { - overpassUpstreamRequests, - overpassUpstreamDuration, -} from "~/lib/metrics.server"; - -const URL_A = "https://a.example/api/interpreter"; -const URL_B = "https://b.example/api/interpreter"; -const URL_C = "https://c.example/api/interpreter"; - -function makeResponse( - body: string, - init: { status?: number; headers?: Record } = {}, -): Response { - return new Response(body, { - status: init.status ?? 200, - headers: init.headers ?? { "content-type": "application/json" }, - }); -} - -beforeEach(() => { - // `clearAllMocks` zeroes call-history on module-level mocks like - // `overpassUpstreamRequests.inc`, which would otherwise accumulate - // across test cases and break assertions that inspect call counts. - vi.clearAllMocks(); -}); - -afterEach(() => { - vi.unstubAllGlobals(); -}); - -describe("fetchWithFailover", () => { - it("returns the first upstream's response when it succeeds", async () => { - const fetchSpy = vi.fn().mockResolvedValueOnce(makeResponse('{"ok":1}')); - vi.stubGlobal("fetch", fetchSpy); - - const result = await fetchWithFailover( - "data=...", - new AbortController().signal, - [URL_A, URL_B], - ); - - expect(result?.status).toBe(200); - expect(result?.body).toBe('{"ok":1}'); - expect(fetchSpy).toHaveBeenCalledTimes(1); - expect(fetchSpy).toHaveBeenCalledWith(URL_A, expect.any(Object)); - expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({ - upstream: "a.example", - status: "200", - }); - }); - - it("falls over to the next upstream on non-2xx", async () => { - const fetchSpy = vi - .fn() - .mockResolvedValueOnce(makeResponse("gateway timeout", { status: 504 })) - .mockResolvedValueOnce(makeResponse('{"elements":[]}')); - vi.stubGlobal("fetch", fetchSpy); - - const result = await fetchWithFailover( - "data=...", - new AbortController().signal, - [URL_A, URL_B], - ); - - expect(result?.body).toBe('{"elements":[]}'); - expect(fetchSpy).toHaveBeenCalledTimes(2); - expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({ - upstream: "a.example", - status: "504", - }); - expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({ - upstream: "b.example", - status: "200", - }); - }); - - it("falls over when the body carries Overpass's rate_limited runtime error", async () => { - // Overpass returns HTTP 200 with a `rate_limited` marker in the - // body when it's throttling. Treat that as an upstream failure. - const fetchSpy = vi - .fn() - .mockResolvedValueOnce(makeResponse('{"error":"rate_limited"}')) - .mockResolvedValueOnce(makeResponse('{"elements":[]}')); - vi.stubGlobal("fetch", fetchSpy); - - const result = await fetchWithFailover( - "data=...", - new AbortController().signal, - [URL_A, URL_B], - ); - - expect(result?.body).toBe('{"elements":[]}'); - expect(fetchSpy).toHaveBeenCalledTimes(2); - }); - - it("falls over on network / timeout errors", async () => { - const fetchSpy = vi - .fn() - .mockRejectedValueOnce(new Error("connect ECONNREFUSED")) - .mockResolvedValueOnce(makeResponse('{"elements":[]}')); - vi.stubGlobal("fetch", fetchSpy); - - const result = await fetchWithFailover( - "data=...", - new AbortController().signal, - [URL_A, URL_B], - ); - - expect(result?.body).toBe('{"elements":[]}'); - expect(fetchSpy).toHaveBeenCalledTimes(2); - expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({ - upstream: "a.example", - status: "error", - }); - }); - - it("tags TimeoutError distinctly from other errors", async () => { - const timeoutErr = new Error("aborted"); - timeoutErr.name = "TimeoutError"; - const fetchSpy = vi - .fn() - .mockRejectedValueOnce(timeoutErr) - .mockResolvedValueOnce(makeResponse('{"elements":[]}')); - vi.stubGlobal("fetch", fetchSpy); - - await fetchWithFailover( - "data=...", - new AbortController().signal, - [URL_A, URL_B], - ); - - expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({ - upstream: "a.example", - status: "timeout", - }); - }); - - it("returns null when every upstream fails", async () => { - const fetchSpy = vi - .fn() - .mockResolvedValueOnce(makeResponse("bad", { status: 500 })) - .mockResolvedValueOnce(makeResponse("bad", { status: 502 })) - .mockResolvedValueOnce(makeResponse("bad", { status: 503 })); - vi.stubGlobal("fetch", fetchSpy); - - const result = await fetchWithFailover( - "data=...", - new AbortController().signal, - [URL_A, URL_B, URL_C], - ); - - expect(result).toBeNull(); - expect(fetchSpy).toHaveBeenCalledTimes(3); - }); - - it("stops iterating when the client aborts, rethrows, and doesn't try later upstreams", async () => { - const controller = new AbortController(); - const abortErr = new DOMException("aborted", "AbortError"); - const fetchSpy = vi.fn().mockImplementationOnce(async () => { - controller.abort(); - throw abortErr; - }); - vi.stubGlobal("fetch", fetchSpy); - - await expect( - fetchWithFailover("data=...", controller.signal, [URL_A, URL_B]), - ).rejects.toBe(abortErr); - - // Only A was attempted; B never got a chance. - expect(fetchSpy).toHaveBeenCalledTimes(1); - expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({ - upstream: "a.example", - status: "client-abort", - }); - }); - - it("records latency with the upstream label on every attempt (success and failure)", async () => { - const fetchSpy = vi - .fn() - .mockResolvedValueOnce(makeResponse("bad", { status: 503 })) - .mockResolvedValueOnce(makeResponse('{"elements":[]}')); - vi.stubGlobal("fetch", fetchSpy); - - await fetchWithFailover( - "data=...", - new AbortController().signal, - [URL_A, URL_B], - ); - - const calls = vi.mocked(overpassUpstreamDuration.observe).mock.calls; - const upstreams = calls.map( - (c) => (c[0] as unknown as { upstream: string }).upstream, - ); - expect(upstreams).toEqual(["a.example", "b.example"]); - }); -}); diff --git a/apps/planner/app/routes/api.overpass.ts b/apps/planner/app/routes/api.overpass.ts deleted file mode 100644 index 64d3226..0000000 --- a/apps/planner/app/routes/api.overpass.ts +++ /dev/null @@ -1,127 +0,0 @@ -import type { Route } from "./+types/api.overpass"; -import { checkRateLimit } from "~/lib/rate-limit"; -import { requireSession } from "~/lib/require-session"; -import { overpassCacheEvents, overpassCacheSize } from "~/lib/metrics.server"; -import { - fetchWithFailover, - type UpstreamResult, -} from "~/lib/overpass.server"; - -const CACHE_TTL_MS = 10 * 60 * 1000; -const CACHE_MAX_ENTRIES = 200; - -interface CacheEntry { - body: string; - contentType: string; - expiresAt: number; -} - -const cache = new Map(); -const inFlight = new Map>(); - -function getFromCache(key: string): CacheEntry | null { - const entry = cache.get(key); - if (!entry) return null; - if (Date.now() > entry.expiresAt) { - cache.delete(key); - return null; - } - // LRU touch: move to end so oldest-first eviction works - cache.delete(key); - cache.set(key, entry); - return entry; -} - -function putInCache(key: string, body: string, contentType: string) { - while (cache.size >= CACHE_MAX_ENTRIES) { - const oldest = cache.keys().next().value; - if (oldest === undefined) break; - cache.delete(oldest); - } - cache.set(key, { body, contentType, expiresAt: Date.now() + CACHE_TTL_MS }); - overpassCacheSize.set(cache.size); -} - -export async function action({ request }: Route.ActionArgs) { - if (request.method !== "POST") { - return new Response("Method not allowed", { status: 405 }); - } - - // Same-origin check. Trust Caddy's X-Forwarded-* headers when present so - // the check works behind the reverse proxy (request.url inside the container - // is http://planner:3001/..., but the browser Origin is https://planner.trails.cool). - const origin = request.headers.get("origin"); - const requestUrl = new URL(request.url); - const forwardedHost = request.headers.get("x-forwarded-host"); - const forwardedProto = request.headers.get("x-forwarded-proto"); - const host = forwardedHost ?? requestUrl.host; - const proto = forwardedProto ?? requestUrl.protocol.replace(":", ""); - const expectedOrigin = `${proto}://${host}`; - if (!origin || origin !== expectedOrigin) { - return new Response("Forbidden", { status: 403 }); - } - - // Session-bind: every proxy call must present a live planner session, - // so anonymous abuse traffic can't ride on our trails.cool Origin. - const session = await requireSession(request.headers.get("x-trails-session")); - if (session instanceof Response) return session; - - const body = await request.text(); - const cacheKey = body; - - // Cache hit: skip rate limit and upstream entirely - const cached = getFromCache(cacheKey); - if (cached) { - overpassCacheEvents.inc({ result: "hit" }); - return new Response(cached.body, { - status: 200, - headers: { "Content-Type": cached.contentType, "X-Cache": "hit" }, - }); - } - - const ip = request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? "unknown"; - const limit = checkRateLimit(`overpass:${ip}`, { maxRequests: 120, windowMs: 60 * 1000 }); - if (!limit.allowed) { - return new Response("Rate limit exceeded", { - status: 429, - headers: { "Retry-After": String(limit.retryAfterSeconds ?? 60) }, - }); - } - - // Coalesce concurrent misses for the same key so multiple clients in the - // same session don't each hit upstream for the identical bbox. - let pending = inFlight.get(cacheKey); - if (!pending) { - overpassCacheEvents.inc({ result: "miss" }); - pending = (async (): Promise => { - try { - const result = await fetchWithFailover(body, request.signal); - if (!result) { - throw new Error("all upstreams failed"); - } - return result; - } finally { - inFlight.delete(cacheKey); - } - })(); - inFlight.set(cacheKey, pending); - } else { - overpassCacheEvents.inc({ result: "coalesced" }); - } - - let result: UpstreamResult; - try { - result = await pending; - } catch { - return new Response("Upstream Overpass unavailable", { status: 502 }); - } - - if (result.cacheable) { - putInCache(cacheKey, result.body, result.contentType); - } - - return new Response(result.body, { - status: result.status, - headers: { "Content-Type": result.contentType, "X-Cache": "miss" }, - }); -} diff --git a/apps/planner/app/routes/api.pois.test.ts b/apps/planner/app/routes/api.pois.test.ts new file mode 100644 index 0000000..47814c0 --- /dev/null +++ b/apps/planner/app/routes/api.pois.test.ts @@ -0,0 +1,159 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +// Keep prom-client (and its DB-collecting gauges) out of these tests; assert +// on the counter labels instead. +vi.mock("~/lib/metrics.server", () => ({ + poiApiRequests: { inc: vi.fn() }, +})); + +vi.mock("~/lib/db.ts", () => ({ getDb: vi.fn() })); +vi.mock("~/lib/require-session.ts", () => ({ requireSession: vi.fn() })); +vi.mock("~/lib/rate-limit.ts", () => ({ checkRateLimit: vi.fn() })); + +import { loader } from "./api.pois.ts"; +import { getDb } from "~/lib/db.ts"; +import { requireSession } from "~/lib/require-session.ts"; +import { checkRateLimit } from "~/lib/rate-limit.ts"; +import { poiApiRequests } from "~/lib/metrics.server"; + +const mockedGetDb = vi.mocked(getDb); +const mockedRequireSession = vi.mocked(requireSession); +const mockedCheckRateLimit = vi.mocked(checkRateLimit); + +// Minimal drizzle chain: select().from().where().limit() resolves to `rows`. +function stubDb(rows: unknown[] | Error) { + const chain = { + select: () => chain, + from: () => chain, + where: () => chain, + limit: () => (rows instanceof Error ? Promise.reject(rows) : Promise.resolve(rows)), + }; + mockedGetDb.mockReturnValue(chain as never); +} + +const VALID_BBOX = "52.50,13.30,52.60,13.50"; + +function makeRequest(query: string, headers: Record = {}): Request { + return new Request(`http://localhost:3001/api/pois?${query}`, { + method: "GET", + headers: { "x-trails-session": "sess-1", ...headers }, + }); +} + +function call(request: Request) { + return loader({ request } as never); +} + +beforeEach(() => { + vi.clearAllMocks(); + // Default: authenticated + under the rate limit. + mockedRequireSession.mockResolvedValue({ id: "sess-1" } as never); + mockedCheckRateLimit.mockReturnValue({ allowed: true, remaining: 100 }); +}); + +describe("api.pois loader", () => { + it("happy path: returns POIs from the index with a short private cache", async () => { + stubDb([ + { + osmId: "123", + category: "drinking_water", + name: "Brunnen", + tags: { amenity: "drinking_water", name: "Brunnen" }, + lat: 52.52, + lon: 13.4, + }, + ]); + + const res = await call(makeRequest(`bbox=${VALID_BBOX}&categories=drinking_water`)); + expect(res.status).toBe(200); + expect(res.headers.get("Cache-Control")).toBe("private, max-age=60"); + + const body = await res.json(); + expect(body.pois).toHaveLength(1); + expect(body.pois[0]).toMatchObject({ + id: 123, + lat: 52.52, + lon: 13.4, + name: "Brunnen", + category: "drinking_water", + }); + expect(poiApiRequests.inc).toHaveBeenCalledWith({ status: "ok" }); + }); + + it("returns an empty list (200) when the viewport has no POIs", async () => { + stubDb([]); + const res = await call(makeRequest(`bbox=${VALID_BBOX}&categories=drinking_water`)); + expect(res.status).toBe(200); + expect((await res.json()).pois).toEqual([]); + }); + + it("400s on a malformed bbox and never queries the DB", async () => { + const res = await call(makeRequest("bbox=not-a-bbox&categories=drinking_water")); + expect(res.status).toBe(400); + expect(mockedGetDb).not.toHaveBeenCalled(); + expect(poiApiRequests.inc).toHaveBeenCalledWith({ status: "bad_request" }); + }); + + it("400s on an out-of-range / inverted bbox", async () => { + // north < south + const res = await call(makeRequest("bbox=52.60,13.30,52.50,13.50&categories=toilets")); + expect(res.status).toBe(400); + }); + + it("400s on an unknown category", async () => { + const res = await call(makeRequest(`bbox=${VALID_BBOX}&categories=drinking_water,banks`)); + expect(res.status).toBe(400); + expect(mockedGetDb).not.toHaveBeenCalled(); + }); + + it("400s when no categories are supplied", async () => { + const res = await call(makeRequest(`bbox=${VALID_BBOX}&categories=`)); + expect(res.status).toBe(400); + }); + + it("429s when rate limited, without executing any DB query", async () => { + mockedCheckRateLimit.mockReturnValue({ allowed: false, remaining: 0, retryAfterSeconds: 42 }); + const res = await call(makeRequest(`bbox=${VALID_BBOX}&categories=drinking_water`)); + expect(res.status).toBe(429); + expect(res.headers.get("Retry-After")).toBe("42"); + expect(mockedGetDb).not.toHaveBeenCalled(); + expect(poiApiRequests.inc).toHaveBeenCalledWith({ status: "rate_limited" }); + }); + + it("401s when there is no valid session (and rate limit not consulted)", async () => { + mockedRequireSession.mockResolvedValue(new Response("Missing session", { status: 401 })); + const res = await call(makeRequest(`bbox=${VALID_BBOX}&categories=drinking_water`, {})); + expect(res.status).toBe(401); + expect(mockedCheckRateLimit).not.toHaveBeenCalled(); + expect(poiApiRequests.inc).toHaveBeenCalledWith({ status: "unauthorized" }); + }); + + it("403s on a cross-origin request", async () => { + const res = await call( + makeRequest(`bbox=${VALID_BBOX}&categories=drinking_water`, { + origin: "https://evil.example", + }), + ); + expect(res.status).toBe(403); + expect(poiApiRequests.inc).toHaveBeenCalledWith({ status: "forbidden" }); + }); + + it("allows a matching same-origin Origin header (via forwarded host/proto)", async () => { + stubDb([]); + const res = await call( + makeRequest(`bbox=${VALID_BBOX}&categories=drinking_water`, { + origin: "https://planner.trails.cool", + "x-forwarded-host": "planner.trails.cool", + "x-forwarded-proto": "https", + }), + ); + expect(res.status).toBe(200); + }); + + it("503s (graceful) when the index table is missing or the DB is down", async () => { + stubDb(new Error('relation "planner.pois" does not exist')); + const res = await call(makeRequest(`bbox=${VALID_BBOX}&categories=drinking_water`)); + expect(res.status).toBe(503); + expect(poiApiRequests.inc).toHaveBeenCalledWith({ status: "error" }); + }); +}); diff --git a/apps/planner/app/routes/api.pois.ts b/apps/planner/app/routes/api.pois.ts new file mode 100644 index 0000000..9e9e88c --- /dev/null +++ b/apps/planner/app/routes/api.pois.ts @@ -0,0 +1,140 @@ +import { and, inArray, sql } from "drizzle-orm"; +import { poiCategories } from "@trails-cool/map-core"; +import { pois } from "@trails-cool/db/schema/planner"; +import type { Route } from "./+types/api.pois"; +import { getDb } from "~/lib/db.ts"; +import { checkRateLimit } from "~/lib/rate-limit.ts"; +import { requireSession } from "~/lib/require-session.ts"; +import { poiApiRequests } from "~/lib/metrics.server.ts"; + +// Same per-IP budget the Overpass proxy enforced — it now protects our own +// database instead of a third-party upstream. +const RATE_LIMIT = { maxRequests: 120, windowMs: 60 * 1000 }; + +// Matches the former Overpass `out ... 100` cap so the client renders the +// same volume of markers per viewport. +const RESULT_CAP = 100; + +const KNOWN_CATEGORY_IDS = new Set(poiCategories.map((c) => c.id)); + +interface PoiResult { + id: number; + lat: number; + lon: number; + name?: string; + category: string; + tags: Record; +} + +function bad(status: number, message: string, metric: string): Response { + poiApiRequests.inc({ status: metric }); + return new Response(message, { status }); +} + +/** + * Parse `south,west,north,east` into a validated bbox. Returns null when the + * value is missing, malformed, out of WGS84 range, or has a non-positive + * extent. + */ +function parseBbox(raw: string | null) { + if (!raw) return null; + const parts = raw.split(",").map(Number); + if (parts.length !== 4 || parts.some((n) => !Number.isFinite(n))) return null; + const [south, west, north, east] = parts as [number, number, number, number]; + if (south < -90 || north > 90 || west < -180 || east > 180) return null; + if (south >= north || west >= east) return null; + return { south, west, north, east }; +} + +export async function loader({ request }: Route.LoaderArgs) { + // Same-origin defense in depth. A cross-origin browser fetch always carries + // an Origin header (even on GET); a same-origin GET may omit it. So reject + // only on a *mismatched* Origin, and lean on the session gate below as the + // primary control. + const requestUrl = new URL(request.url); + const origin = request.headers.get("origin"); + if (origin) { + const host = request.headers.get("x-forwarded-host") ?? requestUrl.host; + const proto = + request.headers.get("x-forwarded-proto") ?? requestUrl.protocol.replace(":", ""); + if (origin !== `${proto}://${host}`) { + return bad(403, "Forbidden", "forbidden"); + } + } + + // Session-bind: every request must present a live planner session so the + // endpoint isn't anonymously reachable on our origin. + const session = await requireSession(request.headers.get("x-trails-session")); + if (session instanceof Response) { + poiApiRequests.inc({ status: "unauthorized" }); + return session; + } + + // Rate limit BEFORE touching the database — rejected requests never query. + const ip = request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? "unknown"; + const limit = checkRateLimit(`pois:${ip}`, RATE_LIMIT); + if (!limit.allowed) { + poiApiRequests.inc({ status: "rate_limited" }); + return new Response("Rate limit exceeded", { + status: 429, + headers: { "Retry-After": String(limit.retryAfterSeconds ?? 60) }, + }); + } + + const bbox = parseBbox(requestUrl.searchParams.get("bbox")); + if (!bbox) return bad(400, "Invalid bbox", "bad_request"); + + const requested = (requestUrl.searchParams.get("categories") ?? "") + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + if (requested.length === 0 || requested.some((c) => !KNOWN_CATEGORY_IDS.has(c))) { + return bad(400, "Invalid categories", "bad_request"); + } + + try { + const rows = await getDb() + .select({ + osmId: pois.osmId, + category: pois.category, + name: pois.name, + tags: pois.tags, + lat: sql`ST_Y(${pois.geom})`, + lon: sql`ST_X(${pois.geom})`, + }) + .from(pois) + .where( + and( + inArray(pois.category, requested), + sql`${pois.geom} && ST_MakeEnvelope(${bbox.west}, ${bbox.south}, ${bbox.east}, ${bbox.north}, 4326)`, + ), + ) + .limit(RESULT_CAP); + + const result: PoiResult[] = rows.map((r) => ({ + id: Number(r.osmId), + lat: r.lat, + lon: r.lon, + name: r.name ?? undefined, + category: r.category, + tags: r.tags ?? {}, + })); + + poiApiRequests.inc({ status: "ok" }); + return new Response(JSON.stringify({ pois: result }), { + status: 200, + headers: { + "Content-Type": "application/json", + // Short private cache: repeated identical viewports (pan back, etc.) + // are served from the browser cache without re-querying. + "Cache-Control": "private, max-age=60", + }, + }); + } catch (err) { + // Missing table (fresh self-hosted instance) or DB outage: degrade + // gracefully so tile overlays keep working and the map doesn't break. + console.error("[api.pois] query failed:", err instanceof Error ? err.message : err); + poiApiRequests.inc({ status: "error" }); + return new Response("POI service unavailable", { status: 503 }); + } +} diff --git a/infrastructure/.env.example b/infrastructure/.env.example index 0c41a05..682aac5 100644 --- a/infrastructure/.env.example +++ b/infrastructure/.env.example @@ -11,9 +11,11 @@ SESSION_SECRET=change-me S3_ACCESS_KEY= S3_SECRET_KEY= -# Overpass upstream failover. Comma-separated list tried in order; -# first 2xx response wins. Falls back to the single-URL OVERPASS_URL -# if this is unset, and to a built-in default list if neither is set. +# Overpass upstream failover for the Journal's surface-breakdown backfill +# (route-surface-breakdown). Comma-separated list tried in order; first 2xx +# response wins. Falls back to the single-URL OVERPASS_URL if unset, and to a +# built-in default list if neither is set. The Planner no longer uses Overpass +# — its POI overlays are served from the self-hosted `planner.pois` index. # OVERPASS_URLS=https://lz4.overpass-api.de/api/interpreter,https://overpass-api.de/api/interpreter # Demo-activity-bot. Only enable in prod. When true, the journal worker diff --git a/infrastructure/docker-compose.staging.yml b/infrastructure/docker-compose.staging.yml index 4e53d26..ce31c5b 100644 --- a/infrastructure/docker-compose.staging.yml +++ b/infrastructure/docker-compose.staging.yml @@ -94,7 +94,6 @@ services: environment: BROUTER_URL: ${BROUTER_URL:?BROUTER_URL must be set} BROUTER_AUTH_TOKEN: ${BROUTER_AUTH_TOKEN:?BROUTER_AUTH_TOKEN must be set} - OVERPASS_URLS: ${OVERPASS_URLS:-https://lz4.overpass-api.de/api/interpreter,https://overpass-api.de/api/interpreter} DATABASE_URL: postgres://trails:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in SOPS secrets.app.env}@postgres:5432/${STAGING_DATABASE} NODE_ENV: production PORT: 3001 diff --git a/infrastructure/docker-compose.yml b/infrastructure/docker-compose.yml index fb599e8..3374bd9 100644 --- a/infrastructure/docker-compose.yml +++ b/infrastructure/docker-compose.yml @@ -103,12 +103,6 @@ services: # BRouter request. The Caddy sidecar on the dedicated host # enforces this header. Set in SOPS secrets.app.env. BROUTER_AUTH_TOKEN: ${BROUTER_AUTH_TOKEN:?BROUTER_AUTH_TOKEN must be set in SOPS secrets.app.env} - # Ordered failover list for the Overpass proxy. The code defaults - # to the same pair if unset; declaring it here makes the prod - # upstream explicit and easy to reshuffle via env when a - # community instance misbehaves. Override per-env with - # OVERPASS_URLS in the SOPS file. - OVERPASS_URLS: ${OVERPASS_URLS:-https://lz4.overpass-api.de/api/interpreter,https://overpass-api.de/api/interpreter} DATABASE_URL: postgres://trails:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in SOPS secrets.app.env}@postgres:5432/trails NODE_ENV: production PORT: 3001 diff --git a/openspec/changes/poi-index/tasks.md b/openspec/changes/poi-index/tasks.md index a571681..71c2f39 100644 --- a/openspec/changes/poi-index/tasks.md +++ b/openspec/changes/poi-index/tasks.md @@ -1,12 +1,12 @@ ## 1. Category selectors (single source of truth) -- [ ] 1.1 Refactor `packages/map-core/src/poi.ts`: replace Overpass QL `query` strings with structured `selectors: Array<{key, value}>` per category; update `poi.test.ts` -- [ ] 1.2 Add a helper that renders the selectors as an osmium tag-filter expression list (used by the pipeline; unit-tested against the nine categories) +- [x] 1.1 Refactor `packages/map-core/src/poi.ts`: replace Overpass QL `query` strings with structured `selectors: Array<{key, value}>` per category; update `poi.test.ts` +- [x] 1.2 Add a helper that renders the selectors as an osmium tag-filter expression list (used by the pipeline; unit-tested against the nine categories) ## 2. Schema -- [ ] 2.1 Add `planner.pois` to `packages/db` (osm_type, osm_id, category, name, geom Point 4326, tags jsonb, imported_at; PK (osm_type, osm_id, category); GiST on geom, btree on category) + migration -- [ ] 2.2 `pnpm db:push` locally and verify bbox+category query plans use the indexes +- [x] 2.1 Add `planner.pois` to `packages/db` (osm_type, osm_id, category, name, geom Point 4326, tags jsonb, imported_at; PK (osm_type, osm_id, category); GiST on geom, btree on category) + migration +- [ ] 2.2 `pnpm db:push` locally and verify bbox+category query plans use the indexes _(needs a local PostGIS; run `pnpm dev:services && pnpm db:push`)_ ## 3. Extract pipeline (BRouter host) @@ -24,18 +24,18 @@ ## 5. Serving route -- [ ] 5.1 Create `apps/planner/app/routes/api.pois.ts` (GET bbox + categories): same-origin + session checks, 120/IP/min rate limit (no DB query on rejection), 100-result cap, short Cache-Control; register in `apps/planner/app/routes.ts` -- [ ] 5.2 Unit tests: happy path, invalid bbox/categories 400, 429 path, empty-table graceful response +- [x] 5.1 Create `apps/planner/app/routes/api.pois.ts` (GET bbox + categories): same-origin + session checks, 120/IP/min rate limit (no DB query on rejection), 100-result cap, short Cache-Control; register in `apps/planner/app/routes.ts` +- [x] 5.2 Unit tests: happy path, invalid bbox/categories 400, 429 path, empty-table graceful response ## 6. Client switch -- [ ] 6.1 Rewrite `apps/planner/app/lib/overpass.ts` as the `/api/pois` client (keep `Poi` shape, `quantizeBbox`, error mapping; drop QL building); rename file to `pois.ts` and update imports (`use-pois`, `use-nearby-pois`, snap-to-poi) -- [ ] 6.2 Update client tests; verify the existing rate-limit banner and unavailable message fire on 429/5xx from `/api/pois` -- [ ] 6.3 Delete `apps/planner/app/routes/api.overpass.ts` and its tests; remove the route registration and the private.coffee configuration +- [x] 6.1 Rewrite `apps/planner/app/lib/overpass.ts` as the `/api/pois` client (keep `Poi` shape, `quantizeBbox`, error mapping; drop QL building); rename file to `pois.ts` and update imports (`use-pois`, `use-nearby-pois`, snap-to-poi) +- [x] 6.2 Update client tests; verify the existing rate-limit banner and unavailable message fire on 429/5xx from `/api/pois` +- [x] 6.3 Delete `apps/planner/app/routes/api.overpass.ts` and its tests; remove the route registration and the private.coffee configuration (`OVERPASS_URLS` kept for the Journal surface backfill; removed from the planner service in both compose files) ## 7. Observability -- [ ] 7.1 Add `poi_index_rows{category}`, `poi_index_age_seconds`, import outcome, and `poi_api_requests_total{status}` metrics; remove `overpass_upstream_*` metrics from `metrics.server.ts` +- [x] 7.1 Add `poi_index_rows{category}`, `poi_index_age_seconds`, import outcome, and `poi_api_requests_total{status}` metrics; remove `overpass_upstream_*` metrics from `metrics.server.ts` (planner exposes rows/age via scrape-time DB collect + `poi_api_requests_total`; import outcome is emitted by the import job via node_exporter textfile — see Task 4) - [ ] 7.2 Update the Grafana planner dashboard: replace Overpass upstream panels with index freshness + serving panels; add the stale-index alert (~6 weeks) ## 8. Docs & cleanup diff --git a/packages/db/src/schema/planner.ts b/packages/db/src/schema/planner.ts index a797ef4..6c52bc3 100644 --- a/packages/db/src/schema/planner.ts +++ b/packages/db/src/schema/planner.ts @@ -1,4 +1,4 @@ -import { pgSchema, text, timestamp, boolean, customType, index } from "drizzle-orm/pg-core"; +import { pgSchema, text, timestamp, boolean, jsonb, customType, index, primaryKey } from "drizzle-orm/pg-core"; const bytea = customType<{ data: Buffer }>({ dataType() { @@ -6,6 +6,15 @@ const bytea = customType<{ data: Buffer }>({ }, }); +// Centroid point in WGS84. Ways/relations are reduced to their centroid at +// extract time (matching the old Overpass `out center`), so every POI is a +// single point regardless of its OSM geometry. +const point = customType<{ data: string }>({ + dataType() { + return "geometry(Point, 4326)"; + }, +}); + export const plannerSchema = pgSchema("planner"); export const sessions = plannerSchema.table("sessions", { @@ -22,3 +31,30 @@ export const sessions = plannerSchema.table("sessions", { // total sessions ever created. lastActivityIdx: index("sessions_last_activity_idx").on(t.lastActivity), })); + +/** + * Self-hosted POI index. Fed monthly by the extract pipeline (osmium filter on + * the BRouter host) + the flagship import job's atomic staging swap — never + * written at request time. Served read-only by `/api/pois` for the planner's + * map overlays, replacing the former Overpass proxy. + * + * An OSM element that matches more than one category is stored once per + * category (part of the composite primary key), because the client treats each + * category's markers independently. + */ +export const pois = plannerSchema.table("pois", { + // OSM element type: 'n' (node), 'w' (way), or 'r' (relation). + osmType: text("osm_type").notNull(), + osmId: text("osm_id").notNull(), + category: text("category").notNull(), + name: text("name"), + geom: point("geom").notNull(), + tags: jsonb("tags").$type>().notNull(), + importedAt: timestamp("imported_at", { withTimezone: true }).notNull().defaultNow(), +}, (t) => ({ + pk: primaryKey({ columns: [t.osmType, t.osmId, t.category] }), + // GiST spatial index for the bbox intersection in the serving query. + geomIdx: index("pois_geom_idx").using("gist", t.geom), + // btree on category so `category = ANY($cats)` is index-driven. + categoryIdx: index("pois_category_idx").on(t.category), +})); diff --git a/packages/map-core/src/index.ts b/packages/map-core/src/index.ts index 821e82e..1495eb0 100644 --- a/packages/map-core/src/index.ts +++ b/packages/map-core/src/index.ts @@ -12,8 +12,14 @@ export { maxspeedColor, } from "./colors/index.ts"; -export type { PoiCategory } from "./poi.ts"; -export { poiCategories, getCategoriesForProfile, profileOverlayDefaults } from "./poi.ts"; +export type { PoiCategory, PoiSelector } from "./poi.ts"; +export { + poiCategories, + getCategoriesForProfile, + matchingCategoryIds, + osmiumTagFilters, + profileOverlayDefaults, +} from "./poi.ts"; export { Z_CURSOR, Z_GHOST_WAYPOINT, Z_WAYPOINT, diff --git a/packages/map-core/src/poi.test.ts b/packages/map-core/src/poi.test.ts index 5da41ea..008352d 100644 --- a/packages/map-core/src/poi.test.ts +++ b/packages/map-core/src/poi.test.ts @@ -1,5 +1,11 @@ import { describe, it, expect } from "vitest"; -import { poiCategories, getCategoriesForProfile, profileOverlayDefaults } from "./poi.ts"; +import { + poiCategories, + getCategoriesForProfile, + matchingCategoryIds, + osmiumTagFilters, + profileOverlayDefaults, +} from "./poi.ts"; describe("poiCategories", () => { it("has expected categories", () => { @@ -16,7 +22,11 @@ describe("poiCategories", () => { expect(cat.name).toBeTruthy(); expect(cat.icon).toBeTruthy(); expect(cat.color).toMatch(/^#[0-9a-fA-F]{6}$/); - expect(cat.query).toBeTruthy(); + expect(cat.selectors.length).toBeGreaterThan(0); + for (const sel of cat.selectors) { + expect(sel.key).toBeTruthy(); + expect(sel.value).toBeTruthy(); + } } }); }); @@ -45,6 +55,63 @@ describe("getCategoriesForProfile", () => { }); }); +describe("matchingCategoryIds", () => { + it("classifies an element by its tags", () => { + expect(matchingCategoryIds({ amenity: "drinking_water" })).toEqual(["drinking_water"]); + expect(matchingCategoryIds({ tourism: "viewpoint", name: "Aussicht" })).toEqual(["viewpoints"]); + }); + + it("returns an empty array for elements matching no category", () => { + expect(matchingCategoryIds({ highway: "primary" })).toEqual([]); + expect(matchingCategoryIds({})).toEqual([]); + }); + + it("returns every matching category when tags satisfy more than one", () => { + // A drinking water point that is also tagged as toilets belongs to both. + const ids = matchingCategoryIds({ amenity: "toilets", drinking_water: "yes" }); + expect(ids).toContain("toilets"); + }); + + it("matches secondary selectors within a category", () => { + expect(matchingCategoryIds({ amenity: "water_point" })).toEqual(["drinking_water"]); + expect(matchingCategoryIds({ tourism: "wilderness_hut" })).toEqual(["shelter"]); + }); +}); + +describe("osmiumTagFilters", () => { + it("renders one nwr expression per distinct key, grouping values", () => { + const filters = osmiumTagFilters(); + // amenity spans several categories; all its values collapse into one expr. + const amenity = filters.find((f) => f.startsWith("nwr/amenity=")); + expect(amenity).toBeDefined(); + expect(amenity).toContain("drinking_water"); + expect(amenity).toContain("toilets"); + expect(amenity).toContain("restaurant"); + // tourism and shop each get their own expression. + expect(filters.some((f) => f.startsWith("nwr/tourism="))).toBe(true); + expect(filters.some((f) => f.startsWith("nwr/shop="))).toBe(true); + }); + + it("covers every selector value across the nine categories", () => { + const filters = osmiumTagFilters(); + for (const cat of poiCategories) { + for (const sel of cat.selectors) { + const expr = filters.find((f) => f.startsWith(`nwr/${sel.key}=`)); + expect(expr, `expected a filter for ${sel.key}`).toBeDefined(); + expect(expr!.split("=")[1]!.split(",")).toContain(sel.value); + } + } + }); + + it("does not repeat a value for the same key", () => { + const filters = osmiumTagFilters(); + for (const f of filters) { + const values = f.split("=")[1]!.split(","); + expect(new Set(values).size).toBe(values.length); + } + }); +}); + describe("profileOverlayDefaults", () => { it("maps fastbike to cycling routes", () => { expect(profileOverlayDefaults["fastbike"]).toContain("waymarked-cycling"); diff --git a/packages/map-core/src/poi.ts b/packages/map-core/src/poi.ts index 26980d2..43e3685 100644 --- a/packages/map-core/src/poi.ts +++ b/packages/map-core/src/poi.ts @@ -1,9 +1,23 @@ +/** + * A single OSM tag selector: an element matches the selector when its tags + * contain `key` mapped to `value`. Categories are unions of selectors. + * + * This structured form is the single source of truth for "what is a shelter": + * the POI extract pipeline renders it to an osmium tag-filter expression, and + * the flagship importer classifies each filtered element back to its + * category/categories. Nothing builds Overpass QL from these any more. + */ +export interface PoiSelector { + key: string; + value: string; +} + export interface PoiCategory { id: string; name: string; icon: string; color: string; - query: string; + selectors: PoiSelector[]; profiles?: string[]; } @@ -13,43 +27,66 @@ export const poiCategories: PoiCategory[] = [ name: "poi.drinkingWater", icon: "\u{1F4A7}", color: "#2563eb", - query: 'nwr["amenity"="drinking_water"];nwr["amenity"="water_point"];', + selectors: [ + { key: "amenity", value: "drinking_water" }, + { key: "amenity", value: "water_point" }, + ], }, { id: "shelter", name: "poi.shelter", icon: "\u{1F6D6}", color: "#8B6D3A", - query: 'nwr["amenity"="shelter"];nwr["tourism"="wilderness_hut"];', + selectors: [ + { key: "amenity", value: "shelter" }, + { key: "tourism", value: "wilderness_hut" }, + ], profiles: ["trekking"], }, { id: "camping", name: "poi.camping", - icon: "\u26FA", + icon: "⛺", color: "#059669", - query: 'nwr["tourism"="camp_site"];nwr["tourism"="caravan_site"];', + selectors: [ + { key: "tourism", value: "camp_site" }, + { key: "tourism", value: "caravan_site" }, + ], }, { id: "food", name: "poi.food", - icon: "\u{1F37D}\uFE0F", + icon: "\u{1F37D}️", color: "#dc2626", - query: 'nwr["amenity"="restaurant"];nwr["amenity"="cafe"];nwr["amenity"="fast_food"];nwr["amenity"="pub"];nwr["amenity"="biergarten"];', + selectors: [ + { key: "amenity", value: "restaurant" }, + { key: "amenity", value: "cafe" }, + { key: "amenity", value: "fast_food" }, + { key: "amenity", value: "pub" }, + { key: "amenity", value: "biergarten" }, + ], }, { id: "groceries", name: "poi.groceries", icon: "\u{1F6D2}", color: "#f97316", - query: 'nwr["shop"="supermarket"];nwr["shop"="convenience"];nwr["shop"="bakery"];', + selectors: [ + { key: "shop", value: "supermarket" }, + { key: "shop", value: "convenience" }, + { key: "shop", value: "bakery" }, + ], }, { id: "bike_infra", name: "poi.bikeInfra", icon: "\u{1F527}", color: "#8b5cf6", - query: 'nwr["amenity"="bicycle_parking"];nwr["amenity"="bicycle_repair_station"];nwr["amenity"="bicycle_rental"];', + selectors: [ + { key: "amenity", value: "bicycle_parking" }, + { key: "amenity", value: "bicycle_repair_station" }, + { key: "amenity", value: "bicycle_rental" }, + ], profiles: ["fastbike", "safety"], }, { @@ -57,14 +94,18 @@ export const poiCategories: PoiCategory[] = [ name: "poi.accommodation", icon: "\u{1F3E8}", color: "#0891b2", - query: 'nwr["tourism"="hotel"];nwr["tourism"="hostel"];nwr["tourism"="guest_house"];', + selectors: [ + { key: "tourism", value: "hotel" }, + { key: "tourism", value: "hostel" }, + { key: "tourism", value: "guest_house" }, + ], }, { id: "viewpoints", name: "poi.viewpoints", - icon: "\u{1F441}\uFE0F", + icon: "\u{1F441}️", color: "#9333ea", - query: 'nwr["tourism"="viewpoint"];', + selectors: [{ key: "tourism", value: "viewpoint" }], profiles: ["trekking"], }, { @@ -72,7 +113,7 @@ export const poiCategories: PoiCategory[] = [ name: "poi.toilets", icon: "\u{1F6BB}", color: "#6b7280", - query: 'nwr["amenity"="toilets"];', + selectors: [{ key: "amenity", value: "toilets" }], }, ]; @@ -82,6 +123,43 @@ export function getCategoriesForProfile(profile: string): string[] { .map((c) => c.id); } +/** + * Return the ids of every category whose selectors match the given OSM tags. + * An element can belong to more than one category. Used by the POI import + * pipeline to classify each osmium-filtered element; the serving layer reads + * the stored category and never needs to re-classify. + */ +export function matchingCategoryIds( + tags: Record, + categories: PoiCategory[] = poiCategories, +): string[] { + return categories + .filter((cat) => cat.selectors.some((s) => tags[s.key] === s.value)) + .map((cat) => cat.id); +} + +/** + * Render the categories' selectors as a deduplicated list of osmium + * `tags-filter` expressions, one per distinct key=value. Each expression + * matches nodes, ways and relations (`nwr/`), matching the previous Overpass + * `nwr[...]` selectors. Values for the same key are grouped so the filter is + * compact, e.g. `nwr/amenity=drinking_water,water_point`. + */ +export function osmiumTagFilters( + categories: PoiCategory[] = poiCategories, +): string[] { + // key -> ordered set of values (first-seen order, deduplicated) + const byKey = new Map(); + for (const cat of categories) { + for (const { key, value } of cat.selectors) { + const values = byKey.get(key) ?? []; + if (!values.includes(value)) values.push(value); + byKey.set(key, values); + } + } + return [...byKey.entries()].map(([key, values]) => `nwr/${key}=${values.join(",")}`); +} + /** Profile -> tile overlay mapping */ export const profileOverlayDefaults: Record = { fastbike: ["waymarked-cycling"], From e703843b9b02c1a6d7ab84fa8fb075c916402fca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= Date: Sun, 12 Jul 2026 22:48:32 +0200 Subject: [PATCH 02/11] poi-index: extract pipeline (BRouter host) + import job (flagship) + e2e - poi-extract.sh: PBF download -> osmium tags-filter -> centroid NDJSON + manifest, published via the Caddy sidecar at vSwitch-only /poi/* - osmium-filters.txt + poi-selectors.sql generated from map-core selectors, guarded by sync tests so poiCategories stays the single source of truth - poi-import.sh: checksum verify -> classify into category rows -> 70% guard (--bootstrap override) -> atomic rename swap; node_exporter textfile metric for import outcome - systemd service+timer units for both halves (monthly, offset) - cd-infra.yml: ship infrastructure/scripts to the flagship - e2e: mock /api/pois instead of /api/overpass Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/cd-infra.yml | 2 +- e2e/fixtures/test.ts | 9 +- e2e/integration.test.ts | 15 +- e2e/planner-overlays.test.ts | 7 +- e2e/planner-routing.test.ts | 7 +- infrastructure/brouter-host/Caddyfile | 22 ++- .../brouter-host/docker-compose.yml | 3 + .../brouter-host/poi-extract/README.md | 92 ++++++++++ .../poi-extract/gen-osmium-filters.ts | 18 ++ .../poi-extract/osmium-filters.txt | 3 + .../poi-extract/poi-extract.service | 23 +++ .../brouter-host/poi-extract/poi-extract.sh | 103 +++++++++++ .../poi-extract/poi-extract.timer | 14 ++ .../brouter-host/poi-extract/to-ndjson.py | 93 ++++++++++ infrastructure/scripts/README.md | 45 +++++ .../scripts/gen-poi-selectors-sql.ts | 36 ++++ infrastructure/scripts/poi-import.service | 19 ++ infrastructure/scripts/poi-import.sh | 162 ++++++++++++++++++ infrastructure/scripts/poi-import.timer | 12 ++ infrastructure/scripts/poi-selectors.sql | 28 +++ openspec/changes/poi-index/tasks.md | 16 +- .../map-core/src/osmium-filters.sync.test.ts | 22 +++ .../src/poi-selectors-sql.sync.test.ts | 27 +++ 23 files changed, 747 insertions(+), 31 deletions(-) create mode 100644 infrastructure/brouter-host/poi-extract/README.md create mode 100644 infrastructure/brouter-host/poi-extract/gen-osmium-filters.ts create mode 100644 infrastructure/brouter-host/poi-extract/osmium-filters.txt create mode 100644 infrastructure/brouter-host/poi-extract/poi-extract.service create mode 100755 infrastructure/brouter-host/poi-extract/poi-extract.sh create mode 100644 infrastructure/brouter-host/poi-extract/poi-extract.timer create mode 100755 infrastructure/brouter-host/poi-extract/to-ndjson.py create mode 100644 infrastructure/scripts/README.md create mode 100644 infrastructure/scripts/gen-poi-selectors-sql.ts create mode 100644 infrastructure/scripts/poi-import.service create mode 100755 infrastructure/scripts/poi-import.sh create mode 100644 infrastructure/scripts/poi-import.timer create mode 100644 infrastructure/scripts/poi-selectors.sql create mode 100644 packages/map-core/src/osmium-filters.sync.test.ts create mode 100644 packages/map-core/src/poi-selectors-sql.sync.test.ts diff --git a/.github/workflows/cd-infra.yml b/.github/workflows/cd-infra.yml index 05f20e6..df41b93 100644 --- a/.github/workflows/cd-infra.yml +++ b/.github/workflows/cd-infra.yml @@ -43,7 +43,7 @@ jobs: host: ${{ secrets.DEPLOY_HOST }} username: root key: ${{ secrets.DEPLOY_SSH_KEY }} - source: "infrastructure/docker-compose.yml,infrastructure/caddy,infrastructure/prometheus/prometheus.yml,infrastructure/loki/loki-config.yml,infrastructure/promtail/promtail-config.yml,infrastructure/postgres/queries.yml,infrastructure/postgres/init-grafana-user.sql,infrastructure/grafana/provisioning,infrastructure/grafana/dashboards" + source: "infrastructure/docker-compose.yml,infrastructure/caddy,infrastructure/prometheus/prometheus.yml,infrastructure/loki/loki-config.yml,infrastructure/promtail/promtail-config.yml,infrastructure/postgres/queries.yml,infrastructure/postgres/init-grafana-user.sql,infrastructure/grafana/provisioning,infrastructure/grafana/dashboards,infrastructure/scripts" target: /opt/trails-cool strip_components: 1 diff --git a/e2e/fixtures/test.ts b/e2e/fixtures/test.ts index ea8c550..4af23cd 100644 --- a/e2e/fixtures/test.ts +++ b/e2e/fixtures/test.ts @@ -8,10 +8,11 @@ import { test as base, expect } from "@playwright/test"; * * The trigger for this safety net was #282 — a POI test that silently * relied on live Overpass data for months because its `page.route` - * was pointing at a URL the browser no longer hit after `/api/overpass` - * was introduced. With this fixture in place, a request to - * `overpass.private.coffee` (or any other unlisted host) would abort - * and the test would surface the missing mock immediately. + * was pointing at a URL the browser no longer hit after the POI proxy + * was introduced. With this fixture in place, a request to any unlisted + * host would abort and the test would surface the missing mock + * immediately. (POIs are now served from the same-origin `/api/pois` + * index endpoint, so the map makes no third-party POI requests at all.) */ const EXTERNAL_ALLOWLIST: RegExp[] = [ // App origins served by Playwright's webServer diff --git a/e2e/integration.test.ts b/e2e/integration.test.ts index 50c6811..059bcb7 100644 --- a/e2e/integration.test.ts +++ b/e2e/integration.test.ts @@ -328,14 +328,15 @@ test.describe("Integration: BRouter routing", () => { expect(response.status()).toBe(401); }); - test("rejects /api/overpass without an X-Trails-Session header", async ({ request }) => { - const response = await request.post(`${PLANNER}/api/overpass`, { - data: "data=[out:json];node[amenity=drinking_water](52.52,13.4,52.53,13.41);out;", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - Origin: PLANNER, + test("rejects /api/pois without an X-Trails-Session header", async ({ request }) => { + const response = await request.get( + `${PLANNER}/api/pois?bbox=52.52,13.40,52.53,13.41&categories=drinking_water`, + { + headers: { + Origin: PLANNER, + }, }, - }); + ); expect(response.status()).toBe(401); }); }); diff --git a/e2e/planner-overlays.test.ts b/e2e/planner-overlays.test.ts index d863a5f..3519cc2 100644 --- a/e2e/planner-overlays.test.ts +++ b/e2e/planner-overlays.test.ts @@ -30,17 +30,18 @@ test.describe("Planner – overlays", () => { test("enable POI category shows markers on map", async ({ page, request }) => { const url = await createSession(request); - await page.route("**/api/overpass", async (route) => { + await page.route("**/api/pois*", async (route) => { await route.fulfill({ status: 200, contentType: "application/json", body: JSON.stringify({ - elements: [ + pois: [ { - type: "node", id: 12345, lat: 52.52, lon: 13.405, + name: "Test Brunnen", + category: "drinking_water", tags: { amenity: "drinking_water", name: "Test Brunnen" }, }, ], diff --git a/e2e/planner-routing.test.ts b/e2e/planner-routing.test.ts index 2b0077e..e1d38af 100644 --- a/e2e/planner-routing.test.ts +++ b/e2e/planner-routing.test.ts @@ -108,17 +108,18 @@ test.describe("Planner – routing", () => { test("nearby POI snap moves waypoint and prepends note prefix", async ({ page, request }) => { const url = await createSession(request); - await page.route("**/api/overpass", async (route) => { + await page.route("**/api/pois*", async (route) => { await route.fulfill({ status: 200, contentType: "application/json", body: JSON.stringify({ - elements: [ + pois: [ { - type: "node", id: 99001, lat: 52.521, lon: 13.406, + name: "Stadtbrunnen", + category: "drinking_water", tags: { amenity: "drinking_water", name: "Stadtbrunnen" }, }, ], diff --git a/infrastructure/brouter-host/Caddyfile b/infrastructure/brouter-host/Caddyfile index b03a095..84536b7 100644 --- a/infrastructure/brouter-host/Caddyfile +++ b/infrastructure/brouter-host/Caddyfile @@ -24,11 +24,23 @@ @authed header X-BRouter-Auth {$BROUTER_AUTH_TOKEN} handle @authed { - reverse_proxy brouter:17777 { - # Don't forward the auth header upstream — BRouter doesn't - # use it, and it's cleaner to contain the credential at - # the proxy boundary. - header_up -X-BRouter-Auth + # POI index artifact + manifest published by poi-extract.sh. Served + # read-only from the mounted publish dir; the flagship import job + # pulls /poi/manifest.json and /poi/pois.ndjson.gz over the vSwitch. + # Same auth + vSwitch-only binding as BRouter, so it's unreachable + # from the public internet. + handle_path /poi/* { + root * /srv/poi + file_server + } + + handle { + reverse_proxy brouter:17777 { + # Don't forward the auth header upstream — BRouter doesn't + # use it, and it's cleaner to contain the credential at + # the proxy boundary. + header_up -X-BRouter-Auth + } } } diff --git a/infrastructure/brouter-host/docker-compose.yml b/infrastructure/brouter-host/docker-compose.yml index 4d82e4f..9958838 100644 --- a/infrastructure/brouter-host/docker-compose.yml +++ b/infrastructure/brouter-host/docker-compose.yml @@ -58,6 +58,9 @@ services: BROUTER_AUTH_TOKEN: ${BROUTER_AUTH_TOKEN:?BROUTER_AUTH_TOKEN must be set} volumes: - ./Caddyfile:/etc/caddy/Caddyfile:ro + # POI index artifact published by poi-extract.sh, served read-only under + # /poi/* on the same vSwitch-only listener as BRouter. + - ./poi-extract/publish:/srv/poi:ro - caddy-data:/data - caddy-config:/config networks: diff --git a/infrastructure/brouter-host/poi-extract/README.md b/infrastructure/brouter-host/poi-extract/README.md new file mode 100644 index 0000000..c59b307 --- /dev/null +++ b/infrastructure/brouter-host/poi-extract/README.md @@ -0,0 +1,92 @@ +# POI extract pipeline (BRouter host) + +Monthly job that turns an OSM PBF into the compact POI artifact the flagship's +`planner.pois` index is built from. This is the "filter where the disk is" +half of the [`poi-index`](../../../openspec/changes/poi-index/) change — it +replaces the third-party Overpass dependency for the Planner's POI overlays. + +Runs on the dedicated BRouter host (`ullrich.is`, the box with 1.8 TB free) +under the non-root `trails` user. The flagship import job +(`infrastructure/scripts/poi-import.sh`) pulls the result over the vSwitch. + +## Files + +| File | Role | +|------|------| +| `poi-extract.sh` | Download PBF → `osmium tags-filter` → `osmium export` → `to-ndjson.py` → gzip + manifest | +| `to-ndjson.py` | GeoJSONSeq → NDJSON, reducing ways/relations to their bbox centre (Overpass `out center` equivalent) | +| `osmium-filters.txt` | osmium tag filters, **generated** from `@trails-cool/map-core` POI selectors | +| `gen-osmium-filters.ts` | Regenerates `osmium-filters.txt` (run after changing categories) | +| `poi-extract.service` / `.timer` | systemd units (monthly) | + +## Single source of truth + +`osmium-filters.txt` is derived from `poiCategories` in +`packages/map-core/src/poi.ts`. After changing categories, regenerate it: + +```bash +npx tsx infrastructure/brouter-host/poi-extract/gen-osmium-filters.ts +``` + +`packages/map-core/src/osmium-filters.sync.test.ts` fails CI if the committed +file drifts from the selectors, so the host never needs a Node runtime. + +## Artifact contract + +`poi-extract.sh` writes to the publish dir (default `./publish`, mounted into +the Caddy sidecar at `/srv/poi`): + +- `pois.ndjson.gz` — one POI per line: + `{osm_type, osm_id, name, lat, lon, tags}`. Category is **not** stored here; + the flagship importer classifies each element from its `tags` using the same + map-core selectors (so the selector logic is never duplicated on this host). +- `manifest.json` — `{artifact, sha256, row_count, generated_at, source}`. + +Caddy serves both under `/poi/*` on the existing vSwitch-only `:17777` +listener, gated by the same `X-BRouter-Auth` shared secret as BRouter — so the +artifact is unreachable from the public internet. + +## Prerequisites + +Install on the host: `osmium-tool`, `python3`, `curl`, `gzip`, `coreutils` +(`sha256sum`). On Debian/Ubuntu: `apt install osmium-tool python3`. + +## Install the timer + +Deployed under `~trails/brouter/poi-extract/`. As a user unit with lingering: + +```bash +loginctl enable-linger trails +mkdir -p ~/.config/systemd/user +cp poi-extract.service poi-extract.timer ~/.config/systemd/user/ +systemctl --user daemon-reload +systemctl --user enable --now poi-extract.timer +systemctl --user list-timers poi-extract.timer +``` + +Trigger a run by hand (e.g. the first bootstrap): `systemctl --user start +poi-extract.service` then `journalctl --user -u poi-extract.service -f`. + +## Footprint + +- **Bandwidth**: the planet PBF is ~80 GB downloaded per run (monthly). Point + `POI_PBF_URL` at a Geofabrik mirror or a regional/continental extract to cut + this — e.g. `europe-latest.osm.pbf` (~30 GB) or `germany-latest.osm.pbf` + (~4 GB) for a smaller instance. +- **Disk**: the planet PBF + filtered PBF live transiently in `./work` and are + removed on exit (even on failure) via a trap. Peak transient usage ≈ size of + the PBF. The published artifact is a few hundred MB gzipped. +- **CPU/time**: a full planet filter is IO-bound and can take a few hours; the + unit runs at `Nice=10` / idle IO priority so it doesn't disturb BRouter. + +## Dry run first + +Before the first planet run, validate the whole chain on a small extract: + +```bash +POI_PBF_URL=https://download.geofabrik.de/europe/germany-latest.osm.pbf \ + ./poi-extract.sh +cat publish/manifest.json # row_count sane? sha256 present? +zcat publish/pois.ndjson.gz | head # records well-formed? +zcat publish/pois.ndjson.gz | wc -l +``` diff --git a/infrastructure/brouter-host/poi-extract/gen-osmium-filters.ts b/infrastructure/brouter-host/poi-extract/gen-osmium-filters.ts new file mode 100644 index 0000000..cd8e7d2 --- /dev/null +++ b/infrastructure/brouter-host/poi-extract/gen-osmium-filters.ts @@ -0,0 +1,18 @@ +/** + * Regenerate `osmium-filters.txt` from the POI category selectors in + * `@trails-cool/map-core` — the single source of truth for "what is a + * shelter". Run this after changing `poiCategories`: + * + * pnpm --filter @trails-cool/map-core exec tsx \ + * ../../infrastructure/brouter-host/poi-extract/gen-osmium-filters.ts + * + * The committed file is what the (Node-free) BRouter host reads at extract + * time. `osmium-filters.sync.test.ts` in map-core fails CI if the two drift. + */ +import { writeFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { osmiumTagFilters } from "../../../packages/map-core/src/poi.ts"; + +const out = fileURLToPath(new URL("./osmium-filters.txt", import.meta.url)); +writeFileSync(out, osmiumTagFilters().join("\n") + "\n"); +console.log(`Wrote ${out}`); diff --git a/infrastructure/brouter-host/poi-extract/osmium-filters.txt b/infrastructure/brouter-host/poi-extract/osmium-filters.txt new file mode 100644 index 0000000..e0d4f16 --- /dev/null +++ b/infrastructure/brouter-host/poi-extract/osmium-filters.txt @@ -0,0 +1,3 @@ +nwr/amenity=drinking_water,water_point,shelter,restaurant,cafe,fast_food,pub,biergarten,bicycle_parking,bicycle_repair_station,bicycle_rental,toilets +nwr/tourism=wilderness_hut,camp_site,caravan_site,hotel,hostel,guest_house,viewpoint +nwr/shop=supermarket,convenience,bakery diff --git a/infrastructure/brouter-host/poi-extract/poi-extract.service b/infrastructure/brouter-host/poi-extract/poi-extract.service new file mode 100644 index 0000000..5b751ce --- /dev/null +++ b/infrastructure/brouter-host/poi-extract/poi-extract.service @@ -0,0 +1,23 @@ +# POI extract — one-shot unit driven by poi-extract.timer. Runs as the +# non-root `trails` user (docker-group, scoped to ~trails/brouter/). Install as +# a user unit: `systemctl --user enable --now poi-extract.timer` with lingering +# enabled (`loginctl enable-linger trails`), or as a system unit with +# User=trails. See README.md. +[Unit] +Description=Filter an OSM PBF to trails.cool POI categories and publish the artifact +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +# Adjust WorkingDirectory to the deployed path (e.g. /home/trails/brouter/poi-extract). +WorkingDirectory=%h/brouter/poi-extract +ExecStart=%h/brouter/poi-extract/poi-extract.sh +# The planet download + filter is long-running and IO-heavy; give it room and +# keep it off the box's foreground priority. +Nice=10 +IOSchedulingClass=idle +TimeoutStartSec=6h + +[Install] +WantedBy=default.target diff --git a/infrastructure/brouter-host/poi-extract/poi-extract.sh b/infrastructure/brouter-host/poi-extract/poi-extract.sh new file mode 100755 index 0000000..5731b0c --- /dev/null +++ b/infrastructure/brouter-host/poi-extract/poi-extract.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# POI extract pipeline — runs on the dedicated BRouter host under the `trails` +# user (the box with 1.8 TB of disk). Downloads an OSM PBF, filters it to the +# planner's POI category selectors with osmium, reduces every element to a +# centroid, and publishes a compact NDJSON artifact + manifest that the +# flagship import job pulls over the vSwitch. +# +# Nothing here is planner/journal code — it's plain osmium + python so the host +# needs no Node runtime. The tag filter comes from `osmium-filters.txt`, which +# is generated from `@trails-cool/map-core`'s POI selectors (single source of +# truth; kept in sync by osmium-filters.sync.test.ts). +# +# Prerequisites on the host: osmium-tool, python3, curl, gzip, sha256sum. +# +# Usage: +# ./poi-extract.sh +# +# Env: +# POI_PBF_URL OSM PBF to filter. Default: the full planet. A self-hoster or +# a dry run can point this at a Geofabrik regional extract, e.g. +# https://download.geofabrik.de/europe/germany-latest.osm.pbf +# POI_WORK_DIR Scratch dir for the (large, transient) planet download. +# Default: