trails-cool/trails
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #562 from trails-cool/poi-index

Browse source 
poi-index: replace Overpass with a self-hosted POI index
This commit is contained in:
Ullrich Schäfer 2026-07-13 00:07:32 +02:00 • committed by GitHub
commit 552899e2b7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
60 changed files with 1865 additions and 1118 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.github/workflows/cd-brouter.yml vendored
View file

@ -87,6 +87,7 @@ jobs:
TARBALL_B64=$(tar -C infrastructure/brouter-host -czf - \
docker-compose.yml Caddyfile promtail-config.yml \
download-segments.sh .env \
poi-extract \
| base64 -w0)
# One SSH session: untar, (re)start containers, report status
@ -99,7 +100,7 @@ jobs:
set -euo pipefail
mkdir -p ~/brouter && cd ~/brouter
echo "$TARBALL_B64" | base64 -d | tar -xzf -
chmod +x download-segments.sh
chmod +x download-segments.sh poi-extract/poi-extract.sh poi-extract/to-ndjson.py
# Segment seeding is a one-shot operator task (~10 GB, a few
# minutes). CD must not rerun it on every deploy.

2
.github/workflows/cd-infra.yml vendored
View file

@ -43,7 +43,7 @@ jobs:
host: ${{ secrets.DEPLOY_HOST }}
username: root
key: ${{ secrets.DEPLOY_SSH_KEY }}
source: "infrastructure/docker-compose.yml,infrastructure/caddy,infrastructure/prometheus/prometheus.yml,infrastructure/loki/loki-config.yml,infrastructure/promtail/promtail-config.yml,infrastructure/postgres/queries.yml,infrastructure/postgres/init-grafana-user.sql,infrastructure/grafana/provisioning,infrastructure/grafana/dashboards"
source: "infrastructure/docker-compose.yml,infrastructure/caddy,infrastructure/prometheus/prometheus.yml,infrastructure/loki/loki-config.yml,infrastructure/promtail/promtail-config.yml,infrastructure/postgres/queries.yml,infrastructure/postgres/init-grafana-user.sql,infrastructure/grafana/provisioning,infrastructure/grafana/dashboards,infrastructure/scripts"
target: /opt/trails-cool
strip_components: 1

28
apps/journal/app/routes/legal.privacy.tsx
View file

@ -253,16 +253,16 @@ export default function PrivacyPage() {
.
</li>
<li>
<strong>Overpass API</strong> (POI- und Wegbelag-Daten) –
POI-Abfragen laufen serverseitig über unsere eigene Route{" "}
<code>/api/overpass</code>. Für die Oberflächen-/Wegtyp-Auswertung
importierter Aktivitäten und Routen fragt ein Hintergrund-Job das
Begrenzungsrechteck (Bounding Box) der Route bei Overpass ab, um sie
den OpenStreetMap-Wegen zuzuordnen. In beiden Fällen sieht der
Upstream nur unsere Server-IP, nicht die Ihrer Nutzer:innen.
Aktueller Upstream: <code>overpass.private.coffee</code> bzw. die
öffentlichen Overpass-Instanzen, die ohne Query-Logs arbeiten. Eine
selbst gehostete Instanz ist geplant.
<strong>Overpass API</strong> (Wegbelag-Daten) – Für die
Oberflächen-/Wegtyp-Auswertung importierter Aktivitäten und Routen
fragt ein Hintergrund-Job das Begrenzungsrechteck (Bounding Box) der
Route bei Overpass ab, um sie den OpenStreetMap-Wegen zuzuordnen.
Der Upstream sieht nur unsere Server-IP, nicht die Ihrer
Nutzer:innen; verwendet werden öffentliche Overpass-Instanzen, die
ohne Query-Logs arbeiten. Die POI-Overlays des Planners nutzen
Overpass <em>nicht</em> mehr – sie werden aus unserem eigenen,
selbst gehosteten POI-Index (<code>/api/pois</code>) bedient, sodass
dabei keine Daten an Dritte gehen.
</li>
<li>
<strong>Föderation (ActivityPub)</strong> – Wenn Ihr Profil auf{" "}
@ -339,9 +339,11 @@ export default function PrivacyPage() {
details, no IPs/cookies); OpenStreetMap tile servers
(your IP and user-agent, directly from your browser, to load map
tiles); Overpass (via our server, so upstream only sees our server —
for POI lookups and, for the surface/waytype breakdown of imported
activities and routes, a background job that sends the route&rsquo;s
bounding box to match it against OpenStreetMap ways); BRouter
for the surface/waytype breakdown of imported activities and routes, a
background job that sends the route&rsquo;s bounding box to match it
against OpenStreetMap ways; the Planner&rsquo;s POI overlays no longer
use Overpass — they are served from our own self-hosted POI index at{" "}
<code>/api/pois</code>); BRouter
(self-hosted, no third party involved); SMTP
provider (your email address for magic link / welcome mail);
Wahoo (only when you opt in: OAuth tokens for sync, plus route

2
apps/planner/app/components/NearbyPoiMarkers.tsx
View file

@ -1,6 +1,6 @@
import L from "leaflet";
import { Marker, Tooltip } from "react-leaflet";
import type { Poi } from "~/lib/overpass";
import type { Poi } from "~/lib/pois";
import type { PoiCategory } from "@trails-cool/map-core";
interface NearbyPoiMarkersProps {

2
apps/planner/app/components/PlannerMap.tsx
View file

@ -29,7 +29,7 @@ import { useGpxDrop } from "~/lib/use-gpx-drop";
import { useNearbyPois } from "~/lib/use-nearby-pois";
import { NearbyPoiMarkers } from "./NearbyPoiMarkers";
import { poiCategories } from "@trails-cool/map-core";
import type { Poi } from "~/lib/overpass";
import type { Poi } from "~/lib/pois";
import "leaflet/dist/leaflet.css";
function waypointIcon(index: number, overnight?: boolean, highlighted?: boolean, hasNote?: boolean): L.DivIcon {

2
apps/planner/app/components/WaypointSidebar.tsx
View file

@ -6,7 +6,7 @@ import { setOvernight } from "~/lib/overnight";
import { DayBreakdown } from "./DayBreakdown";
import { useNearbyPois } from "~/lib/use-nearby-pois";
import { poiCategories } from "@trails-cool/map-core";
import type { Poi } from "~/lib/overpass";
import type { Poi } from "~/lib/pois";
import {
extractWaypointData,
waypointFromYMap,

72
apps/planner/app/lib/metrics.server.ts
View file

@ -1,4 +1,7 @@
import client from "prom-client";
import { sql } from "drizzle-orm";
import { pois } from "@trails-cool/db/schema/planner";
import { getDb } from "./db.ts";
// Guard all metric registration — Vite's dev server can re-evaluate
// this module, causing "already registered" errors.
@ -41,39 +44,58 @@ export const brouterRequestDuration = getOrCreate("brouter_request_duration_seco
}),
);
export const overpassCacheEvents = getOrCreate("overpass_cache_events_total", () =>
// POI serving endpoint (`/api/pois`) request counter. `status` is the
// outcome class: ok | rate_limited | bad_request | forbidden | unauthorized
// | error. Replaces the former Overpass proxy/upstream metrics now that POIs
// are served from the local index.
export const poiApiRequests = getOrCreate("poi_api_requests_total", () =>
new client.Counter({
name: "overpass_cache_events_total",
help: "Overpass proxy cache events",
labelNames: ["result"] as const, // hit | miss | coalesced
name: "poi_api_requests_total",
help: "POI serving endpoint requests by outcome status",
labelNames: ["status"] as const,
}),
);
export const overpassCacheSize = getOrCreate("overpass_cache_size", () =>
// Index size per category, computed from the planner's own DB at scrape time
// (prom-client binds `this` to the gauge inside `collect`).
export const poiIndexRows = getOrCreate("poi_index_rows", () =>
new client.Gauge({
name: "overpass_cache_size",
help: "Current number of entries in the Overpass proxy cache",
name: "poi_index_rows",
help: "Number of POIs in the local index per category",
labelNames: ["category"] as const,
async collect() {
try {
const rows = await getDb()
.select({ category: pois.category, count: sql<number>`count(*)::int` })
.from(pois)
.groupBy(pois.category);
this.reset();
for (const r of rows) this.set({ category: r.category }, r.count);
} catch {
// DB unavailable or table not yet created — leave last known values.
}
},
}),
);
export const overpassUpstreamDuration = getOrCreate("overpass_upstream_duration_seconds", () =>
new client.Histogram({
name: "overpass_upstream_duration_seconds",
help: "Duration of upstream Overpass API requests in seconds",
labelNames: ["upstream"] as const,
// Buckets go to 30s because our per-upstream timeout is 10s; a bit
// of headroom catches slow-but-not-timed-out tails without overly
// coarse resolution at the fast end where lz4 typically lands
// (~100–500ms).
buckets: [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 30],
}),
);
export const overpassUpstreamRequests = getOrCreate("overpass_upstream_requests_total", () =>
new client.Counter({
name: "overpass_upstream_requests_total",
help: "Upstream Overpass API requests by upstream host and status",
labelNames: ["upstream", "status"] as const,
// Age of the freshest imported POI row, in seconds. Drives the stale-index
// alert (~6 weeks = one missed monthly refresh).
export const poiIndexAgeSeconds = getOrCreate("poi_index_age_seconds", () =>
new client.Gauge({
name: "poi_index_age_seconds",
help: "Seconds since the most recent POI index import",
async collect() {
try {
const [row] = await getDb()
.select({
age: sql<number | null>`extract(epoch from (now() - max(${pois.importedAt})))::float`,
})
.from(pois);
if (row?.age != null) this.set(row.age);
} catch {
// DB unavailable or empty index — leave last known value.
}
},
}),
);

111
apps/planner/app/lib/overpass.server.ts
View file

@ -1,111 +0,0 @@
import {
overpassUpstreamDuration,
overpassUpstreamRequests,
} from "~/lib/metrics.server";
/**
* Ordered list of upstream Overpass endpoints. We try each in turn
* until one returns a usable response; on timeout, network error,
* non-2xx status, or a body carrying Overpass's `rate_limited` runtime
* error, we fall over to the next. The default list keeps us on
* healthy community instances; `OVERPASS_URLS` overrides it and
* `OVERPASS_URL` is kept as a single-entry backward-compat alias.
*/
const DEFAULT_UPSTREAMS = [
"https://lz4.overpass-api.de/api/interpreter",
"https://overpass-api.de/api/interpreter",
];
function loadUpstreams(): string[] {
const list = process.env.OVERPASS_URLS ?? process.env.OVERPASS_URL;
if (!list) return DEFAULT_UPSTREAMS;
return list.split(",").map((s) => s.trim()).filter(Boolean);
}
export const UPSTREAMS = loadUpstreams();
const USER_AGENT = "trails.cool Planner (https://trails.cool; legal@trails.cool)";
// Per-attempt wall-clock budget. Keep aggressive so failover kicks in
// quickly when an upstream is saturated — a single slow instance
// shouldn't cost the user minutes.
const PER_UPSTREAM_TIMEOUT_MS = 10_000;
export interface UpstreamResult {
body: string;
contentType: string;
status: number;
cacheable: boolean;
}
function hostOf(url: string): string {
try {
return new URL(url).hostname;
} catch {
return "unknown";
}
}
/**
* Try each upstream in order until one returns a usable response.
* Returns the first successful result, or `null` if every upstream
* failed. `clientSignal` is the browser's abort signal — if the user
* gives up before we succeed, we stop trying and propagate the abort
* to the in-flight upstream fetch so we don't waste its capacity on a
* request nobody wants anymore.
*/
export async function fetchWithFailover(
body: string,
clientSignal: AbortSignal,
urls: readonly string[] = UPSTREAMS,
): Promise<UpstreamResult | null> {
for (const url of urls) {
if (clientSignal.aborted) break;
const upstream = hostOf(url);
const start = Date.now();
try {
const signal = AbortSignal.any([
clientSignal,
AbortSignal.timeout(PER_UPSTREAM_TIMEOUT_MS),
]);
const response = await fetch(url, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"User-Agent": USER_AGENT,
},
body,
signal,
});
const responseBody = await response.text();
overpassUpstreamDuration.observe({ upstream }, (Date.now() - start) / 1000);
overpassUpstreamRequests.inc({ upstream, status: String(response.status) });
// Overpass returns 200 with a `rate_limited` runtime error when
// the instance is throttling us. Treat that like a 5xx: try the
// next upstream.
if (!response.ok || responseBody.includes("rate_limited")) {
continue;
}
return {
body: responseBody,
contentType: response.headers.get("content-type") ?? "application/json",
status: 200,
cacheable: true,
};
} catch (err) {
overpassUpstreamDuration.observe({ upstream }, (Date.now() - start) / 1000);
// If the CLIENT aborted, stop the whole loop — the user doesn't
// want an answer anymore. Any other error (timeout, network,
// DNS) counts as "this upstream failed, try the next."
if (clientSignal.aborted) {
overpassUpstreamRequests.inc({ upstream, status: "client-abort" });
throw err;
}
const label = (err as Error).name === "TimeoutError" ? "timeout" : "error";
overpassUpstreamRequests.inc({ upstream, status: label });
continue;
}
}
return null;
}

199
apps/planner/app/lib/overpass.test.ts
View file

@ -1,199 +0,0 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import { buildQuery, parseResponse, deduplicateById, quantizeBbox, fetchNearbyPois, type Poi } from "./overpass.ts";
import { poiCategories } from "@trails-cool/map-core";
describe("buildQuery", () => {
it("builds Overpass QL with bbox and category union", () => {
const bbox = { south: 52.5, west: 13.3, north: 52.6, east: 13.5 };
const categories = poiCategories.filter((c) => c.id === "drinking_water");
const query = buildQuery(bbox, categories);
expect(query).toContain("[out:json]");
expect(query).toContain("[bbox:52.500,13.300,52.600,13.500]");
expect(query).toContain('amenity"="drinking_water"');
expect(query).toContain("out center qt 100");
});
it("combines multiple categories into a single union", () => {
const bbox = { south: 52.5, west: 13.3, north: 52.6, east: 13.5 };
const categories = poiCategories.filter((c) => ["drinking_water", "camping"].includes(c.id));
const query = buildQuery(bbox, categories);
expect(query).toContain("drinking_water");
expect(query).toContain("camp_site");
});
it("produces identical queries for near-identical viewports within one grid cell", () => {
const categories = poiCategories.filter((c) => c.id === "drinking_water");
const a = buildQuery(
{ south: 52.5041, west: 13.3072, north: 52.5931, east: 13.4928 },
categories,
);
const b = buildQuery(
{ south: 52.5039, west: 13.3078, north: 52.5925, east: 13.4921 },
categories,
);
expect(a).toBe(b);
});
it("expands the bbox outward so the original viewport is always covered", () => {
const categories = poiCategories.filter((c) => c.id === "drinking_water");
const query = buildQuery(
{ south: 52.5041, west: 13.3072, north: 52.5931, east: 13.4928 },
categories,
);
expect(query).toContain("[bbox:52.500,13.300,52.600,13.500]");
});
});
describe("quantizeBbox", () => {
it("snaps south and west down, north and east up", () => {
const q = quantizeBbox({ south: 52.5041, west: 13.3072, north: 52.5931, east: 13.4928 }, 0.01);
expect(q.south).toBeCloseTo(52.5, 10);
expect(q.west).toBeCloseTo(13.3, 10);
expect(q.north).toBeCloseTo(52.6, 10);
expect(q.east).toBeCloseTo(13.5, 10);
});
it("is idempotent on already-aligned coordinates", () => {
const bbox = { south: 52.5, west: 13.3, north: 52.6, east: 13.5 };
const q = quantizeBbox(bbox, 0.01);
expect(q.south).toBeCloseTo(52.5, 10);
expect(q.north).toBeCloseTo(52.6, 10);
});
});
describe("parseResponse", () => {
it("parses nodes into Poi objects", () => {
const data = {
elements: [
{
type: "node",
id: 123,
lat: 52.52,
lon: 13.405,
tags: { amenity: "drinking_water", name: "Brunnen" },
},
],
};
const categories = poiCategories.filter((c) => c.id === "drinking_water");
const pois = parseResponse(data, categories);
expect(pois).toHaveLength(1);
expect(pois[0]!.id).toBe(123);
expect(pois[0]!.name).toBe("Brunnen");
expect(pois[0]!.category).toBe("drinking_water");
expect(pois[0]!.lat).toBe(52.52);
});
it("uses center for way/relation elements", () => {
const data = {
elements: [
{
type: "way",
id: 456,
center: { lat: 52.51, lon: 13.39 },
tags: { tourism: "camp_site", name: "Zeltplatz" },
},
],
};
const categories = poiCategories.filter((c) => c.id === "camping");
const pois = parseResponse(data, categories);
expect(pois).toHaveLength(1);
expect(pois[0]!.lat).toBe(52.51);
expect(pois[0]!.lon).toBe(13.39);
});
it("skips elements without coordinates", () => {
const data = {
elements: [
{ type: "node", id: 789, tags: { amenity: "drinking_water" } },
],
};
const categories = poiCategories.filter((c) => c.id === "drinking_water");
const pois = parseResponse(data, categories);
expect(pois).toHaveLength(0);
});
it("skips elements that don't match any category", () => {
const data = {
elements: [
{ type: "node", id: 100, lat: 52.52, lon: 13.4, tags: { amenity: "bank" } },
],
};
const categories = poiCategories.filter((c) => c.id === "drinking_water");
const pois = parseResponse(data, categories);
expect(pois).toHaveLength(0);
});
});
describe("deduplicateById", () => {
it("removes duplicate POIs by id", () => {
const pois: Poi[] = [
{ id: 1, lat: 52.5, lon: 13.4, category: "drinking_water", tags: {} },
{ id: 1, lat: 52.5, lon: 13.4, category: "camping", tags: {} },
{ id: 2, lat: 52.6, lon: 13.5, category: "shelter", tags: {} },
];
const result = deduplicateById(pois);
expect(result).toHaveLength(2);
expect(result[0]!.id).toBe(1);
expect(result[1]!.id).toBe(2);
});
});
describe("fetchNearbyPois", () => {
const emptyOverpassResponse = { elements: [] };
beforeEach(() => {
vi.stubGlobal("fetch", vi.fn().mockResolvedValue({
ok: true,
text: async () => JSON.stringify(emptyOverpassResponse),
json: async () => emptyOverpassResponse,
}));
});
afterEach(() => {
vi.unstubAllGlobals();
});
it("constructs a bbox from lat/lon/radius and calls fetch", async () => {
const categories = poiCategories.filter((c) => c.id === "drinking_water");
await fetchNearbyPois(50.0, 10.0, 500, categories);
expect(fetch).toHaveBeenCalledOnce();
const rawBody = (((fetch as ReturnType<typeof vi.fn>).mock.calls[0] as unknown[])[1]) as unknown as { body: string };
const body = decodeURIComponent(rawBody.body.replace("data=", ""));
// bbox quantized to 0.01° grid; 500m at lat=50 well within one cell
expect(body).toMatch(/\[bbox:4\d\.\d+,\d+\.\d+,50\.\d+,10\.\d+\]/);
});
it("bbox is symmetric around the input point", async () => {
const lat = 48.0;
const lon = 11.0;
const radius = 1000;
const categories = poiCategories.filter((c) => c.id === "drinking_water");
await fetchNearbyPois(lat, lon, radius, categories);
const rawBody = (((fetch as ReturnType<typeof vi.fn>).mock.calls[0] as unknown[])[1]) as unknown as { body: string };
const body = decodeURIComponent(rawBody.body.replace("data=", ""));
const match = body.match(/\[bbox:([\d.]+),([\d.]+),([\d.]+),([\d.]+)\]/);
expect(match).not.toBeNull();
const [, south, west, north, east] = (match as RegExpMatchArray).map(Number);
// After quantization the center might not be exactly lat/lon, but the extent should be ≥ radius
const DEG_PER_METER_LAT = 1 / 111320;
const dLat = radius * DEG_PER_METER_LAT;
expect((north as number) - (south as number)).toBeGreaterThanOrEqual(dLat * 2 - 0.02);
expect((east as number) - (west as number)).toBeGreaterThan(0);
});
it("forwards AbortSignal to fetch", async () => {
const controller = new AbortController();
const categories = poiCategories.filter((c) => c.id === "drinking_water");
await fetchNearbyPois(50.0, 10.0, 500, categories, controller.signal);
expect(fetch).toHaveBeenCalledOnce();
const callOptions = (((fetch as ReturnType<typeof vi.fn>).mock.calls[0] as unknown[])[1]) as unknown as { signal: AbortSignal };
expect(callOptions?.signal).toBe(controller.signal);
});
});

211
apps/planner/app/lib/overpass.ts
View file

@ -1,211 +0,0 @@
import type { PoiCategory } from "@trails-cool/map-core";
const OVERPASS_PROXY = "/api/overpass";
// Snap bbox coordinates to this grid before building the query so that two
// users looking at nearly-identical viewports produce byte-identical queries
// and therefore share a server-side cache entry. Expansion is outward
// (south/west floor, north/east ceil) so the user's viewport is always
// covered. ~0.01° ≈ 1 km at mid-latitudes.
const BBOX_GRID_STEP = 0.01;
// Decimals used when formatting the quantized bbox into the query string.
// 3 decimals → 0.001° precision (~111 m) which is well below BBOX_GRID_STEP,
// so the string is a stable representation of the quantized cell.
const BBOX_DECIMALS = 3;
export interface Poi {
id: number;
lat: number;
lon: number;
name?: string;
category: string;
tags: Record<string, string>;
}
export interface BBox {
south: number;
west: number;
north: number;
east: number;
}
/**
* Quantize a bbox to the grid defined by `BBOX_GRID_STEP`, expanding outward
* so the original bbox is fully contained. Returns a new BBox whose
* coordinates are cell-aligned.
*/
export function quantizeBbox(bbox: BBox, step: number = BBOX_GRID_STEP): BBox {
return {
south: Math.floor(bbox.south / step) * step,
west: Math.floor(bbox.west / step) * step,
north: Math.ceil(bbox.north / step) * step,
east: Math.ceil(bbox.east / step) * step,
};
}
/**
* Build an Overpass QL query combining all enabled categories into a union.
* The bbox is quantized to a fixed grid and formatted with a fixed number of
* decimals so near-identical viewports produce a byte-identical query — which
* the `/api/overpass` server-side cache keys on.
*/
export function buildQuery(bbox: BBox, categories: PoiCategory[]): string {
const q = quantizeBbox(bbox);
const bboxStr = [q.south, q.west, q.north, q.east]
.map((n) => n.toFixed(BBOX_DECIMALS))
.join(",");
const unions = categories.map((c) => c.query).join("");
return `[out:json][timeout:10][maxsize:1048576][bbox:${bboxStr}];(${unions});out center qt 100;`;
}
/**
* Parse Overpass JSON response into typed Poi objects.
*/
export function parseResponse(
data: { elements: Array<{
type: string;
id: number;
lat?: number;
lon?: number;
center?: { lat: number; lon: number };
tags?: Record<string, string>;
}> },
categories: PoiCategory[],
): Poi[] {
const pois: Poi[] = [];
for (const el of data.elements) {
const lat = el.lat ?? el.center?.lat;
const lon = el.lon ?? el.center?.lon;
if (lat === undefined || lon === undefined) continue;
const tags = el.tags ?? {};
const category = matchCategory(tags, categories);
if (!category) continue;
pois.push({
id: el.id,
lat,
lon,
name: tags.name,
category: category.id,
tags,
});
}
return deduplicateById(pois);
}
/**
* Match an element's tags to the first matching category.
*/
function matchCategory(tags: Record<string, string>, categories: PoiCategory[]): PoiCategory | null {
for (const cat of categories) {
// Parse query fragments like 'nwr["amenity"="drinking_water"];'
const fragments = cat.query.split(";").filter(Boolean);
for (const frag of fragments) {
const match = frag.match(/\["(\w+)"="([^"]+)"\]/);
if (match && tags[match[1]!] === match[2]) return cat;
}
}
return null;
}
/**
* Deduplicate POIs by OSM node ID (same node may match multiple queries).
*/
export function deduplicateById(pois: Poi[]): Poi[] {
const seen = new Set<number>();
return pois.filter((poi) => {
if (seen.has(poi.id)) return false;
seen.add(poi.id);
return true;
});
}
/**
* Query the Overpass API for POIs within a bounding box.
*
* All queries route through the Planner's own `/api/overpass` proxy, which
* adds a User-Agent identifying trails.cool, rate-limits per IP, and enforces
* same-origin. The client never talks to a public Overpass host directly.
*/
export async function queryPois(
bbox: BBox,
categories: PoiCategory[],
sessionId: string,
signal?: AbortSignal,
): Promise<Poi[]> {
if (categories.length === 0) return [];
const query = buildQuery(bbox, categories);
const response = await fetch(OVERPASS_PROXY, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
// Bind this call to the active planner session so the proxy
// isn't anonymously reachable.
"X-Trails-Session": sessionId,
},
body: `data=${encodeURIComponent(query)}`,
signal,
});
if (response.status === 429) {
throw new OverpassRateLimitError();
}
if (!response.ok) {
throw new Error(`Overpass API error: ${response.status}`);
}
const text = await response.text();
if (text.includes("rate_limited")) {
throw new OverpassRateLimitError();
}
let data;
try {
data = JSON.parse(text);
} catch {
throw new Error("Overpass API returned invalid JSON");
}
return parseResponse(data, categories);
}
export class OverpassRateLimitError extends Error {
constructor() {
super("Overpass API rate limit exceeded");
this.name = "OverpassRateLimitError";
}
}
// Degrees of latitude per meter (approximate, valid globally).
const DEG_PER_METER_LAT = 1 / 111320;
/**
* Fetch POIs within `radiusMeters` of a coordinate.
* Builds a bbox around the point and reuses queryPois + the existing proxy.
*/
export async function fetchNearbyPois(
lat: number,
lon: number,
radiusMeters: number,
categories: import("@trails-cool/map-core").PoiCategory[],
signal?: AbortSignal,
sessionId = "nearby",
): Promise<Poi[]> {
const dLat = radiusMeters * DEG_PER_METER_LAT;
const dLon = dLat / Math.cos((lat * Math.PI) / 180);
const bbox: BBox = {
south: lat - dLat,
north: lat + dLat,
west: lon - dLon,
east: lon + dLon,
};
return queryPois(bbox, categories, sessionId, signal);
}

2
apps/planner/app/lib/poi-cache.test.ts
View file

@ -1,6 +1,6 @@
import { describe, it, expect, beforeEach } from "vitest";
import { quantize, getCellKeys, getCached, setCached, clearCache } from "./poi-cache.ts";
import type { Poi } from "./overpass.ts";
import type { Poi } from "./pois.ts";
beforeEach(() => {
clearCache();

2
apps/planner/app/lib/poi-cache.ts
View file

@ -1,4 +1,4 @@
import type { Poi, BBox } from "./overpass.ts";
import type { Poi, BBox } from "./pois.ts";
const CELL_SIZE = 0.1; // degrees
const TTL = 10 * 60 * 1000; // 10 minutes

2
apps/planner/app/lib/poi-categories.test.ts
View file

@ -40,7 +40,7 @@ describe("poiCategories", () => {
expect(cat.name).toBeTruthy();
expect(cat.icon).toBeTruthy();
expect(cat.color).toMatch(/^#/);
expect(cat.query).toContain("nwr");
expect(cat.selectors.length).toBeGreaterThan(0);
}
});
});

2
apps/planner/app/lib/poi-snap.ts
View file

@ -1,4 +1,4 @@
import type { Poi } from "./overpass.ts";
import type { Poi } from "./pois.ts";
import { SNAP_DISTANCE_METERS } from "@trails-cool/map-core";
export interface SnapResult {

160
apps/planner/app/lib/pois.test.ts Normal file
View file

@ -0,0 +1,160 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import {
buildPoisQuery,
quantizeBbox,
deduplicateById,
queryPois,
fetchNearbyPois,
PoiRateLimitError,
type Poi,
} from "./pois.ts";
import { poiCategories } from "@trails-cool/map-core";
const drinkingWater = poiCategories.filter((c) => c.id === "drinking_water");
describe("buildPoisQuery", () => {
it("encodes the quantized bbox and category ids", () => {
const bbox = { south: 52.5, west: 13.3, north: 52.6, east: 13.5 };
const q = buildPoisQuery(bbox, drinkingWater);
expect(q).toContain("bbox=52.500,13.300,52.600,13.500");
expect(q).toContain("categories=drinking_water");
});
it("joins multiple categories", () => {
const cats = poiCategories.filter((c) => ["drinking_water", "camping"].includes(c.id));
const q = buildPoisQuery({ south: 52.5, west: 13.3, north: 52.6, east: 13.5 }, cats);
expect(decodeURIComponent(q)).toContain("categories=drinking_water,camping");
});
it("produces identical query strings for near-identical viewports in one cell", () => {
const a = buildPoisQuery(
{ south: 52.5041, west: 13.3072, north: 52.5931, east: 13.4928 },
drinkingWater,
);
const b = buildPoisQuery(
{ south: 52.5039, west: 13.3078, north: 52.5925, east: 13.4921 },
drinkingWater,
);
expect(a).toBe(b);
});
});
describe("quantizeBbox", () => {
it("snaps south/west down and north/east up", () => {
const q = quantizeBbox({ south: 52.5041, west: 13.3072, north: 52.5931, east: 13.4928 }, 0.01);
expect(q.south).toBeCloseTo(52.5, 10);
expect(q.west).toBeCloseTo(13.3, 10);
expect(q.north).toBeCloseTo(52.6, 10);
expect(q.east).toBeCloseTo(13.5, 10);
});
});
describe("deduplicateById", () => {
it("keeps the first occurrence of each id", () => {
const pois: Poi[] = [
{ id: 1, lat: 52.5, lon: 13.4, category: "drinking_water", tags: {} },
{ id: 1, lat: 52.5, lon: 13.4, category: "camping", tags: {} },
{ id: 2, lat: 52.6, lon: 13.5, category: "shelter", tags: {} },
];
const result = deduplicateById(pois);
expect(result.map((p) => p.id)).toEqual([1, 2]);
});
});
describe("queryPois", () => {
afterEach(() => vi.unstubAllGlobals());
it("GETs /api/pois with the session header and parses the response", async () => {
const fetchSpy = vi.fn().mockResolvedValue({
ok: true,
status: 200,
json: async () => ({
pois: [{ id: 7, lat: 52.5, lon: 13.4, name: "Brunnen", category: "drinking_water", tags: {} }],
}),
});
vi.stubGlobal("fetch", fetchSpy);
const pois = await queryPois(
{ south: 52.5, west: 13.3, north: 52.6, east: 13.5 },
drinkingWater,
"sess-1",
);
expect(pois).toHaveLength(1);
expect(pois[0]!.id).toBe(7);
const [url, opts] = fetchSpy.mock.calls[0] as [string, RequestInit];
expect(url).toContain("/api/pois?bbox=");
expect(url).toContain("categories=drinking_water");
expect((opts.headers as Record<string, string>)["X-Trails-Session"]).toBe("sess-1");
expect(opts.method).toBe("GET");
});
it("returns [] without fetching when no categories are enabled", async () => {
const fetchSpy = vi.fn();
vi.stubGlobal("fetch", fetchSpy);
const pois = await queryPois({ south: 52.5, west: 13.3, north: 52.6, east: 13.5 }, [], "s");
expect(pois).toEqual([]);
expect(fetchSpy).not.toHaveBeenCalled();
});
it("throws PoiRateLimitError on a 429", async () => {
vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: false, status: 429 }));
await expect(
queryPois({ south: 52.5, west: 13.3, north: 52.6, east: 13.5 }, drinkingWater, "s"),
).rejects.toBeInstanceOf(PoiRateLimitError);
});
it("throws a generic error on other non-ok statuses (e.g. 503)", async () => {
vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: false, status: 503 }));
await expect(
queryPois({ south: 52.5, west: 13.3, north: 52.6, east: 13.5 }, drinkingWater, "s"),
).rejects.toThrow("POI service error: 503");
});
it("deduplicates POIs returned across categories", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockResolvedValue({
ok: true,
status: 200,
json: async () => ({
pois: [
{ id: 1, lat: 52.5, lon: 13.4, category: "drinking_water", tags: {} },
{ id: 1, lat: 52.5, lon: 13.4, category: "toilets", tags: {} },
],
}),
}),
);
const pois = await queryPois(
{ south: 52.5, west: 13.3, north: 52.6, east: 13.5 },
drinkingWater,
"s",
);
expect(pois).toHaveLength(1);
});
});
describe("fetchNearbyPois", () => {
beforeEach(() => {
vi.stubGlobal(
"fetch",
vi.fn().mockResolvedValue({ ok: true, status: 200, json: async () => ({ pois: [] }) }),
);
});
afterEach(() => vi.unstubAllGlobals());
it("builds a bbox around the point and calls /api/pois", async () => {
await fetchNearbyPois(50.0, 10.0, 500, drinkingWater);
expect(fetch).toHaveBeenCalledOnce();
const [url] = (fetch as ReturnType<typeof vi.fn>).mock.calls[0] as [string];
expect(url).toMatch(/\/api\/pois\?bbox=\d+\.\d+,\d+\.\d+,\d+\.\d+,\d+\.\d+/);
expect(url).toContain("categories=drinking_water");
});
it("forwards the AbortSignal to fetch", async () => {
const controller = new AbortController();
await fetchNearbyPois(50.0, 10.0, 500, drinkingWater, controller.signal);
const [, opts] = (fetch as ReturnType<typeof vi.fn>).mock.calls[0] as [string, RequestInit];
expect(opts.signal).toBe(controller.signal);
});
});

148
apps/planner/app/lib/pois.ts Normal file
View file

@ -0,0 +1,148 @@
import type { PoiCategory } from "@trails-cool/map-core";
const POIS_ENDPOINT = "/api/pois";
// Snap bbox coordinates to this grid before requesting so that two users
// looking at nearly-identical viewports produce the same URL and can share a
// browser-cache entry. Expansion is outward (south/west floor, north/east
// ceil) so the user's viewport is always covered. ~0.01° ≈ 1 km at
// mid-latitudes.
const BBOX_GRID_STEP = 0.01;
// Decimals used when formatting the quantized bbox into the request URL.
// 3 decimals → 0.001° precision (~111 m), well below BBOX_GRID_STEP, so the
// string is a stable representation of the quantized cell.
const BBOX_DECIMALS = 3;
export interface Poi {
id: number;
lat: number;
lon: number;
name?: string;
category: string;
tags: Record<string, string>;
}
export interface BBox {
south: number;
west: number;
north: number;
east: number;
}
/**
* Quantize a bbox to the grid defined by `BBOX_GRID_STEP`, expanding outward
* so the original bbox is fully contained. Returns a new BBox whose
* coordinates are cell-aligned.
*/
export function quantizeBbox(bbox: BBox, step: number = BBOX_GRID_STEP): BBox {
return {
south: Math.floor(bbox.south / step) * step,
west: Math.floor(bbox.west / step) * step,
north: Math.ceil(bbox.north / step) * step,
east: Math.ceil(bbox.east / step) * step,
};
}
/**
* Build the `/api/pois` query string for a bbox + categories. The bbox is
* quantized and formatted with fixed decimals so near-identical viewports
* produce a byte-identical URL — which the browser cache keys on.
*/
export function buildPoisQuery(bbox: BBox, categories: PoiCategory[]): string {
const q = quantizeBbox(bbox);
const bboxStr = [q.south, q.west, q.north, q.east]
.map((n) => n.toFixed(BBOX_DECIMALS))
.join(",");
const cats = categories.map((c) => c.id).join(",");
return `bbox=${bboxStr}&categories=${encodeURIComponent(cats)}`;
}
/**
* Deduplicate POIs by OSM id (an element in two enabled categories comes back
* once per category; the map shows a single marker for it).
*/
export function deduplicateById(pois: Poi[]): Poi[] {
const seen = new Set<number>();
return pois.filter((poi) => {
if (seen.has(poi.id)) return false;
seen.add(poi.id);
return true;
});
}
/**
* Query the instance's own POI index for POIs within a bounding box.
*
* All requests go to the Planner's `/api/pois` endpoint, which enforces
* same-origin + session binding and a per-IP rate limit, and serves from the
* local PostGIS index. No POI request ever leaves trails.cool infrastructure.
*/
export async function queryPois(
bbox: BBox,
categories: PoiCategory[],
sessionId: string,
signal?: AbortSignal,
): Promise<Poi[]> {
if (categories.length === 0) return [];
const response = await fetch(`${POIS_ENDPOINT}?${buildPoisQuery(bbox, categories)}`, {
method: "GET",
headers: {
// Bind this call to the active planner session so the endpoint isn't
// anonymously reachable.
"X-Trails-Session": sessionId,
},
signal,
});
if (response.status === 429) {
throw new PoiRateLimitError();
}
if (!response.ok) {
throw new Error(`POI service error: ${response.status}`);
}
let data: { pois?: Poi[] };
try {
data = await response.json();
} catch {
throw new Error("POI service returned invalid JSON");
}
return deduplicateById(data.pois ?? []);
}
export class PoiRateLimitError extends Error {
constructor() {
super("POI service rate limit exceeded");
this.name = "PoiRateLimitError";
}
}
// Degrees of latitude per meter (approximate, valid globally).
const DEG_PER_METER_LAT = 1 / 111320;
/**
* Fetch POIs within `radiusMeters` of a coordinate.
* Builds a bbox around the point and reuses queryPois + the `/api/pois` route.
*/
export async function fetchNearbyPois(
lat: number,
lon: number,
radiusMeters: number,
categories: PoiCategory[],
signal?: AbortSignal,
sessionId = "nearby",
): Promise<Poi[]> {
const dLat = radiusMeters * DEG_PER_METER_LAT;
const dLon = dLat / Math.cos((lat * Math.PI) / 180);
const bbox: BBox = {
south: lat - dLat,
north: lat + dLat,
west: lon - dLon,
east: lon + dLon,
};
return queryPois(bbox, categories, sessionId, signal);
}

2
apps/planner/app/lib/snap-to-poi.test.ts
View file

@ -1,7 +1,7 @@
import { describe, it, expect } from "vitest";
import * as Y from "yjs";
import { poiCategories } from "@trails-cool/map-core";
import type { Poi } from "./overpass.ts";
import type { Poi } from "./pois.ts";
// Mirrors the snapToPoi logic from WaypointSidebar / PlannerMap
function snapToPoi(

4
apps/planner/app/lib/use-nearby-pois.ts
View file

@ -1,5 +1,5 @@
import { useState, useEffect, useRef } from "react";
import { fetchNearbyPois, OverpassRateLimitError, type Poi } from "./overpass.ts";
import { fetchNearbyPois, PoiRateLimitError, type Poi } from "./pois.ts";
import { poiCategories } from "@trails-cool/map-core";
const NEARBY_RADIUS_METERS = 500;
@ -53,7 +53,7 @@ export function useNearbyPois(
}
} catch (err) {
if (controller.signal.aborted) return;
if (err instanceof OverpassRateLimitError) {
if (err instanceof PoiRateLimitError) {
rateLimitedUntilRef.current = Date.now() + RATE_LIMIT_SUPPRESS_MS;
setState({ pois: [], status: "rate_limited" });
} else {

4
apps/planner/app/lib/use-pois.ts
View file

@ -1,5 +1,5 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { queryPois, OverpassRateLimitError, type Poi, type BBox } from "./overpass.ts";
import { queryPois, PoiRateLimitError, type Poi, type BBox } from "./pois.ts";
import { getCached, setCached } from "./poi-cache.ts";
import { poiCategories } from "@trails-cool/map-core";
@ -86,7 +86,7 @@ export function usePois(sessionId: string): PoiState {
} catch (err) {
if (controller.signal.aborted) return;
if (err instanceof OverpassRateLimitError) {
if (err instanceof PoiRateLimitError) {
setStatus("rate_limited");
backoffRef.current = Math.min(
(backoffRef.current || BACKOFF_BASE_MS) * 2,

2
apps/planner/app/routes.ts
View file

@ -6,7 +6,7 @@ export default [
route("api/sessions", "routes/api.sessions.ts"),
route("api/route", "routes/api.route.ts"),
route("api/route-segments", "routes/api.route-segments.ts"),
route("api/overpass", "routes/api.overpass.ts"),
route("api/pois", "routes/api.pois.ts"),
route("api/save-to-journal", "routes/api.save-to-journal.ts"),
route("session/:id", "routes/session.$id.tsx"),
] satisfies RouteConfig;

209
apps/planner/app/routes/api.overpass.test.ts
View file

@ -1,209 +0,0 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
// Metrics module pulls in prom-client and registers processes-wide
// metrics on import; mock it out so tests aren't entangled with
// the registry and can assert on label values instead.
vi.mock("~/lib/metrics.server", () => ({
overpassCacheEvents: { inc: vi.fn() },
overpassCacheSize: { set: vi.fn() },
overpassUpstreamDuration: { observe: vi.fn() },
overpassUpstreamRequests: { inc: vi.fn() },
}));
import { fetchWithFailover } from "~/lib/overpass.server";
import {
overpassUpstreamRequests,
overpassUpstreamDuration,
} from "~/lib/metrics.server";
const URL_A = "https://a.example/api/interpreter";
const URL_B = "https://b.example/api/interpreter";
const URL_C = "https://c.example/api/interpreter";
function makeResponse(
body: string,
init: { status?: number; headers?: Record<string, string> } = {},
): Response {
return new Response(body, {
status: init.status ?? 200,
headers: init.headers ?? { "content-type": "application/json" },
});
}
beforeEach(() => {
// `clearAllMocks` zeroes call-history on module-level mocks like
// `overpassUpstreamRequests.inc`, which would otherwise accumulate
// across test cases and break assertions that inspect call counts.
vi.clearAllMocks();
});
afterEach(() => {
vi.unstubAllGlobals();
});
describe("fetchWithFailover", () => {
it("returns the first upstream's response when it succeeds", async () => {
const fetchSpy = vi.fn().mockResolvedValueOnce(makeResponse('{"ok":1}'));
vi.stubGlobal("fetch", fetchSpy);
const result = await fetchWithFailover(
"data=...",
new AbortController().signal,
[URL_A, URL_B],
);
expect(result?.status).toBe(200);
expect(result?.body).toBe('{"ok":1}');
expect(fetchSpy).toHaveBeenCalledTimes(1);
expect(fetchSpy).toHaveBeenCalledWith(URL_A, expect.any(Object));
expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({
upstream: "a.example",
status: "200",
});
});
it("falls over to the next upstream on non-2xx", async () => {
const fetchSpy = vi
.fn()
.mockResolvedValueOnce(makeResponse("gateway timeout", { status: 504 }))
.mockResolvedValueOnce(makeResponse('{"elements":[]}'));
vi.stubGlobal("fetch", fetchSpy);
const result = await fetchWithFailover(
"data=...",
new AbortController().signal,
[URL_A, URL_B],
);
expect(result?.body).toBe('{"elements":[]}');
expect(fetchSpy).toHaveBeenCalledTimes(2);
expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({
upstream: "a.example",
status: "504",
});
expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({
upstream: "b.example",
status: "200",
});
});
it("falls over when the body carries Overpass's rate_limited runtime error", async () => {
// Overpass returns HTTP 200 with a `rate_limited` marker in the
// body when it's throttling. Treat that as an upstream failure.
const fetchSpy = vi
.fn()
.mockResolvedValueOnce(makeResponse('{"error":"rate_limited"}'))
.mockResolvedValueOnce(makeResponse('{"elements":[]}'));
vi.stubGlobal("fetch", fetchSpy);
const result = await fetchWithFailover(
"data=...",
new AbortController().signal,
[URL_A, URL_B],
);
expect(result?.body).toBe('{"elements":[]}');
expect(fetchSpy).toHaveBeenCalledTimes(2);
});
it("falls over on network / timeout errors", async () => {
const fetchSpy = vi
.fn()
.mockRejectedValueOnce(new Error("connect ECONNREFUSED"))
.mockResolvedValueOnce(makeResponse('{"elements":[]}'));
vi.stubGlobal("fetch", fetchSpy);
const result = await fetchWithFailover(
"data=...",
new AbortController().signal,
[URL_A, URL_B],
);
expect(result?.body).toBe('{"elements":[]}');
expect(fetchSpy).toHaveBeenCalledTimes(2);
expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({
upstream: "a.example",
status: "error",
});
});
it("tags TimeoutError distinctly from other errors", async () => {
const timeoutErr = new Error("aborted");
timeoutErr.name = "TimeoutError";
const fetchSpy = vi
.fn()
.mockRejectedValueOnce(timeoutErr)
.mockResolvedValueOnce(makeResponse('{"elements":[]}'));
vi.stubGlobal("fetch", fetchSpy);
await fetchWithFailover(
"data=...",
new AbortController().signal,
[URL_A, URL_B],
);
expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({
upstream: "a.example",
status: "timeout",
});
});
it("returns null when every upstream fails", async () => {
const fetchSpy = vi
.fn()
.mockResolvedValueOnce(makeResponse("bad", { status: 500 }))
.mockResolvedValueOnce(makeResponse("bad", { status: 502 }))
.mockResolvedValueOnce(makeResponse("bad", { status: 503 }));
vi.stubGlobal("fetch", fetchSpy);
const result = await fetchWithFailover(
"data=...",
new AbortController().signal,
[URL_A, URL_B, URL_C],
);
expect(result).toBeNull();
expect(fetchSpy).toHaveBeenCalledTimes(3);
});
it("stops iterating when the client aborts, rethrows, and doesn't try later upstreams", async () => {
const controller = new AbortController();
const abortErr = new DOMException("aborted", "AbortError");
const fetchSpy = vi.fn().mockImplementationOnce(async () => {
controller.abort();
throw abortErr;
});
vi.stubGlobal("fetch", fetchSpy);
await expect(
fetchWithFailover("data=...", controller.signal, [URL_A, URL_B]),
).rejects.toBe(abortErr);
// Only A was attempted; B never got a chance.
expect(fetchSpy).toHaveBeenCalledTimes(1);
expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({
upstream: "a.example",
status: "client-abort",
});
});
it("records latency with the upstream label on every attempt (success and failure)", async () => {
const fetchSpy = vi
.fn()
.mockResolvedValueOnce(makeResponse("bad", { status: 503 }))
.mockResolvedValueOnce(makeResponse('{"elements":[]}'));
vi.stubGlobal("fetch", fetchSpy);
await fetchWithFailover(
"data=...",
new AbortController().signal,
[URL_A, URL_B],
);
const calls = vi.mocked(overpassUpstreamDuration.observe).mock.calls;
const upstreams = calls.map(
(c) => (c[0] as unknown as { upstream: string }).upstream,
);
expect(upstreams).toEqual(["a.example", "b.example"]);
});
});

127
apps/planner/app/routes/api.overpass.ts
View file

@ -1,127 +0,0 @@
import type { Route } from "./+types/api.overpass";
import { checkRateLimit } from "~/lib/rate-limit";
import { requireSession } from "~/lib/require-session";
import { overpassCacheEvents, overpassCacheSize } from "~/lib/metrics.server";
import {
fetchWithFailover,
type UpstreamResult,
} from "~/lib/overpass.server";
const CACHE_TTL_MS = 10 * 60 * 1000;
const CACHE_MAX_ENTRIES = 200;
interface CacheEntry {
body: string;
contentType: string;
expiresAt: number;
}
const cache = new Map<string, CacheEntry>();
const inFlight = new Map<string, Promise<UpstreamResult>>();
function getFromCache(key: string): CacheEntry | null {
const entry = cache.get(key);
if (!entry) return null;
if (Date.now() > entry.expiresAt) {
cache.delete(key);
return null;
}
// LRU touch: move to end so oldest-first eviction works
cache.delete(key);
cache.set(key, entry);
return entry;
}
function putInCache(key: string, body: string, contentType: string) {
while (cache.size >= CACHE_MAX_ENTRIES) {
const oldest = cache.keys().next().value;
if (oldest === undefined) break;
cache.delete(oldest);
}
cache.set(key, { body, contentType, expiresAt: Date.now() + CACHE_TTL_MS });
overpassCacheSize.set(cache.size);
}
export async function action({ request }: Route.ActionArgs) {
if (request.method !== "POST") {
return new Response("Method not allowed", { status: 405 });
}
// Same-origin check. Trust Caddy's X-Forwarded-* headers when present so
// the check works behind the reverse proxy (request.url inside the container
// is http://planner:3001/..., but the browser Origin is https://planner.trails.cool).
const origin = request.headers.get("origin");
const requestUrl = new URL(request.url);
const forwardedHost = request.headers.get("x-forwarded-host");
const forwardedProto = request.headers.get("x-forwarded-proto");
const host = forwardedHost ?? requestUrl.host;
const proto = forwardedProto ?? requestUrl.protocol.replace(":", "");
const expectedOrigin = `${proto}://${host}`;
if (!origin || origin !== expectedOrigin) {
return new Response("Forbidden", { status: 403 });
}
// Session-bind: every proxy call must present a live planner session,
// so anonymous abuse traffic can't ride on our trails.cool Origin.
const session = await requireSession(request.headers.get("x-trails-session"));
if (session instanceof Response) return session;
const body = await request.text();
const cacheKey = body;
// Cache hit: skip rate limit and upstream entirely
const cached = getFromCache(cacheKey);
if (cached) {
overpassCacheEvents.inc({ result: "hit" });
return new Response(cached.body, {
status: 200,
headers: { "Content-Type": cached.contentType, "X-Cache": "hit" },
});
}
const ip = request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? "unknown";
const limit = checkRateLimit(`overpass:${ip}`, { maxRequests: 120, windowMs: 60 * 1000 });
if (!limit.allowed) {
return new Response("Rate limit exceeded", {
status: 429,
headers: { "Retry-After": String(limit.retryAfterSeconds ?? 60) },
});
}
// Coalesce concurrent misses for the same key so multiple clients in the
// same session don't each hit upstream for the identical bbox.
let pending = inFlight.get(cacheKey);
if (!pending) {
overpassCacheEvents.inc({ result: "miss" });
pending = (async (): Promise<UpstreamResult> => {
try {
const result = await fetchWithFailover(body, request.signal);
if (!result) {
throw new Error("all upstreams failed");
}
return result;
} finally {
inFlight.delete(cacheKey);
}
})();
inFlight.set(cacheKey, pending);
} else {
overpassCacheEvents.inc({ result: "coalesced" });
}
let result: UpstreamResult;
try {
result = await pending;
} catch {
return new Response("Upstream Overpass unavailable", { status: 502 });
}
if (result.cacheable) {
putInCache(cacheKey, result.body, result.contentType);
}
return new Response(result.body, {
status: result.status,
headers: { "Content-Type": result.contentType, "X-Cache": "miss" },
});
}

159
apps/planner/app/routes/api.pois.test.ts Normal file
View file

@ -0,0 +1,159 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
// Keep prom-client (and its DB-collecting gauges) out of these tests; assert
// on the counter labels instead.
vi.mock("~/lib/metrics.server", () => ({
poiApiRequests: { inc: vi.fn() },
}));
vi.mock("~/lib/db.ts", () => ({ getDb: vi.fn() }));
vi.mock("~/lib/require-session.ts", () => ({ requireSession: vi.fn() }));
vi.mock("~/lib/rate-limit.ts", () => ({ checkRateLimit: vi.fn() }));
import { loader } from "./api.pois.ts";
import { getDb } from "~/lib/db.ts";
import { requireSession } from "~/lib/require-session.ts";
import { checkRateLimit } from "~/lib/rate-limit.ts";
import { poiApiRequests } from "~/lib/metrics.server";
const mockedGetDb = vi.mocked(getDb);
const mockedRequireSession = vi.mocked(requireSession);
const mockedCheckRateLimit = vi.mocked(checkRateLimit);
// Minimal drizzle chain: select().from().where().limit() resolves to `rows`.
function stubDb(rows: unknown[] | Error) {
const chain = {
select: () => chain,
from: () => chain,
where: () => chain,
limit: () => (rows instanceof Error ? Promise.reject(rows) : Promise.resolve(rows)),
};
mockedGetDb.mockReturnValue(chain as never);
}
const VALID_BBOX = "52.50,13.30,52.60,13.50";
function makeRequest(query: string, headers: Record<string, string> = {}): Request {
return new Request(`http://localhost:3001/api/pois?${query}`, {
method: "GET",
headers: { "x-trails-session": "sess-1", ...headers },
});
}
function call(request: Request) {
return loader({ request } as never);
}
beforeEach(() => {
vi.clearAllMocks();
// Default: authenticated + under the rate limit.
mockedRequireSession.mockResolvedValue({ id: "sess-1" } as never);
mockedCheckRateLimit.mockReturnValue({ allowed: true, remaining: 100 });
});
describe("api.pois loader", () => {
it("happy path: returns POIs from the index with a short private cache", async () => {
stubDb([
{
osmId: "123",
category: "drinking_water",
name: "Brunnen",
tags: { amenity: "drinking_water", name: "Brunnen" },
lat: 52.52,
lon: 13.4,
},
]);
const res = await call(makeRequest(`bbox=${VALID_BBOX}&categories=drinking_water`));
expect(res.status).toBe(200);
expect(res.headers.get("Cache-Control")).toBe("private, max-age=60");
const body = await res.json();
expect(body.pois).toHaveLength(1);
expect(body.pois[0]).toMatchObject({
id: 123,
lat: 52.52,
lon: 13.4,
name: "Brunnen",
category: "drinking_water",
});
expect(poiApiRequests.inc).toHaveBeenCalledWith({ status: "ok" });
});
it("returns an empty list (200) when the viewport has no POIs", async () => {
stubDb([]);
const res = await call(makeRequest(`bbox=${VALID_BBOX}&categories=drinking_water`));
expect(res.status).toBe(200);
expect((await res.json()).pois).toEqual([]);
});
it("400s on a malformed bbox and never queries the DB", async () => {
const res = await call(makeRequest("bbox=not-a-bbox&categories=drinking_water"));
expect(res.status).toBe(400);
expect(mockedGetDb).not.toHaveBeenCalled();
expect(poiApiRequests.inc).toHaveBeenCalledWith({ status: "bad_request" });
});
it("400s on an out-of-range / inverted bbox", async () => {
// north < south
const res = await call(makeRequest("bbox=52.60,13.30,52.50,13.50&categories=toilets"));
expect(res.status).toBe(400);
});
it("400s on an unknown category", async () => {
const res = await call(makeRequest(`bbox=${VALID_BBOX}&categories=drinking_water,banks`));
expect(res.status).toBe(400);
expect(mockedGetDb).not.toHaveBeenCalled();
});
it("400s when no categories are supplied", async () => {
const res = await call(makeRequest(`bbox=${VALID_BBOX}&categories=`));
expect(res.status).toBe(400);
});
it("429s when rate limited, without executing any DB query", async () => {
mockedCheckRateLimit.mockReturnValue({ allowed: false, remaining: 0, retryAfterSeconds: 42 });
const res = await call(makeRequest(`bbox=${VALID_BBOX}&categories=drinking_water`));
expect(res.status).toBe(429);
expect(res.headers.get("Retry-After")).toBe("42");
expect(mockedGetDb).not.toHaveBeenCalled();
expect(poiApiRequests.inc).toHaveBeenCalledWith({ status: "rate_limited" });
});
it("401s when there is no valid session (and rate limit not consulted)", async () => {
mockedRequireSession.mockResolvedValue(new Response("Missing session", { status: 401 }));
const res = await call(makeRequest(`bbox=${VALID_BBOX}&categories=drinking_water`, {}));
expect(res.status).toBe(401);
expect(mockedCheckRateLimit).not.toHaveBeenCalled();
expect(poiApiRequests.inc).toHaveBeenCalledWith({ status: "unauthorized" });
});
it("403s on a cross-origin request", async () => {
const res = await call(
makeRequest(`bbox=${VALID_BBOX}&categories=drinking_water`, {
origin: "https://evil.example",
}),
);
expect(res.status).toBe(403);
expect(poiApiRequests.inc).toHaveBeenCalledWith({ status: "forbidden" });
});
it("allows a matching same-origin Origin header (via forwarded host/proto)", async () => {
stubDb([]);
const res = await call(
makeRequest(`bbox=${VALID_BBOX}&categories=drinking_water`, {
origin: "https://planner.trails.cool",
"x-forwarded-host": "planner.trails.cool",
"x-forwarded-proto": "https",
}),
);
expect(res.status).toBe(200);
});
it("503s (graceful) when the index table is missing or the DB is down", async () => {
stubDb(new Error('relation "planner.pois" does not exist'));
const res = await call(makeRequest(`bbox=${VALID_BBOX}&categories=drinking_water`));
expect(res.status).toBe(503);
expect(poiApiRequests.inc).toHaveBeenCalledWith({ status: "error" });
});
});

140
apps/planner/app/routes/api.pois.ts Normal file
View file

@ -0,0 +1,140 @@
import { and, inArray, sql } from "drizzle-orm";
import { poiCategories } from "@trails-cool/map-core";
import { pois } from "@trails-cool/db/schema/planner";
import type { Route } from "./+types/api.pois";
import { getDb } from "~/lib/db.ts";
import { checkRateLimit } from "~/lib/rate-limit.ts";
import { requireSession } from "~/lib/require-session.ts";
import { poiApiRequests } from "~/lib/metrics.server.ts";
// Same per-IP budget the Overpass proxy enforced — it now protects our own
// database instead of a third-party upstream.
const RATE_LIMIT = { maxRequests: 120, windowMs: 60 * 1000 };
// Matches the former Overpass `out ... 100` cap so the client renders the
// same volume of markers per viewport.
const RESULT_CAP = 100;
const KNOWN_CATEGORY_IDS = new Set(poiCategories.map((c) => c.id));
interface PoiResult {
id: number;
lat: number;
lon: number;
name?: string;
category: string;
tags: Record<string, string>;
}
function bad(status: number, message: string, metric: string): Response {
poiApiRequests.inc({ status: metric });
return new Response(message, { status });
}
/**
* Parse `south,west,north,east` into a validated bbox. Returns null when the
* value is missing, malformed, out of WGS84 range, or has a non-positive
* extent.
*/
function parseBbox(raw: string | null) {
if (!raw) return null;
const parts = raw.split(",").map(Number);
if (parts.length !== 4 || parts.some((n) => !Number.isFinite(n))) return null;
const [south, west, north, east] = parts as [number, number, number, number];
if (south < -90 || north > 90 || west < -180 || east > 180) return null;
if (south >= north || west >= east) return null;
return { south, west, north, east };
}
export async function loader({ request }: Route.LoaderArgs) {
// Same-origin defense in depth. A cross-origin browser fetch always carries
// an Origin header (even on GET); a same-origin GET may omit it. So reject
// only on a *mismatched* Origin, and lean on the session gate below as the
// primary control.
const requestUrl = new URL(request.url);
const origin = request.headers.get("origin");
if (origin) {
const host = request.headers.get("x-forwarded-host") ?? requestUrl.host;
const proto =
request.headers.get("x-forwarded-proto") ?? requestUrl.protocol.replace(":", "");
if (origin !== `${proto}://${host}`) {
return bad(403, "Forbidden", "forbidden");
}
}
// Session-bind: every request must present a live planner session so the
// endpoint isn't anonymously reachable on our origin.
const session = await requireSession(request.headers.get("x-trails-session"));
if (session instanceof Response) {
poiApiRequests.inc({ status: "unauthorized" });
return session;
}
// Rate limit BEFORE touching the database — rejected requests never query.
const ip = request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? "unknown";
const limit = checkRateLimit(`pois:${ip}`, RATE_LIMIT);
if (!limit.allowed) {
poiApiRequests.inc({ status: "rate_limited" });
return new Response("Rate limit exceeded", {
status: 429,
headers: { "Retry-After": String(limit.retryAfterSeconds ?? 60) },
});
}
const bbox = parseBbox(requestUrl.searchParams.get("bbox"));
if (!bbox) return bad(400, "Invalid bbox", "bad_request");
const requested = (requestUrl.searchParams.get("categories") ?? "")
.split(",")
.map((s) => s.trim())
.filter(Boolean);
if (requested.length === 0 || requested.some((c) => !KNOWN_CATEGORY_IDS.has(c))) {
return bad(400, "Invalid categories", "bad_request");
}
try {
const rows = await getDb()
.select({
osmId: pois.osmId,
category: pois.category,
name: pois.name,
tags: pois.tags,
lat: sql<number>`ST_Y(${pois.geom})`,
lon: sql<number>`ST_X(${pois.geom})`,
})
.from(pois)
.where(
and(
inArray(pois.category, requested),
sql`${pois.geom} && ST_MakeEnvelope(${bbox.west}, ${bbox.south}, ${bbox.east}, ${bbox.north}, 4326)`,
),
)
.limit(RESULT_CAP);
const result: PoiResult[] = rows.map((r) => ({
id: Number(r.osmId),
lat: r.lat,
lon: r.lon,
name: r.name ?? undefined,
category: r.category,
tags: r.tags ?? {},
}));
poiApiRequests.inc({ status: "ok" });
return new Response(JSON.stringify({ pois: result }), {
status: 200,
headers: {
"Content-Type": "application/json",
// Short private cache: repeated identical viewports (pan back, etc.)
// are served from the browser cache without re-querying.
"Cache-Control": "private, max-age=60",
},
});
} catch (err) {
// Missing table (fresh self-hosted instance) or DB outage: degrade
// gracefully so tile overlays keep working and the map doesn't break.
console.error("[api.pois] query failed:", err instanceof Error ? err.message : err);
poiApiRequests.inc({ status: "error" });
return new Response("POI service unavailable", { status: 503 });
}
}

8
docs/architecture.md
View file

@ -790,8 +790,12 @@ Base layers:
Overlays (toggleable):
- **OpenCampingMap** (campsites, shelters — essential for bikepacking)
- **POI overlay**: Water points, shelters, bike repair stations
(sourced from OSM Overpass API or pre-cached)
- **POI overlay**: Water points, shelters, bike repair stations, etc. Served
from the instance's own `planner.pois` index via `/api/pois` — a PostGIS
table built monthly from an OSM extract (osmium filter on the BRouter host →
guarded atomic import on the flagship). No third-party POI service is
contacted; viewport coordinates never leave trails.cool. See the
`poi-index` change and `infrastructure/brouter-host/poi-extract/`.
- **Waymarked Trails** (hiking/cycling trail networks)
Implementation: Leaflet layer switcher with tile URLs. No API keys needed

23
docs/ideas/self-host-overpass/README.md
View file

@ -1,4 +1,4 @@
# self-host-overpass (parked)
# self-host-overpass (parked — superseded for POIs)
Full OpenSpec artifact set (`proposal.md`, `design.md`, `specs/`, `tasks.md`)
for hosting our own Overpass API. Moved here from `openspec/changes/` so it
@ -7,11 +7,22 @@ under `openspec/changes/` when ready to implement.
## Status
**Parked.** The interim proxy work (`/api/overpass` → `overpass.private.coffee`,
see `apps/planner/app/routes/api.overpass.ts`) covers the day-one needs:
User-Agent compliance, same-origin check, rate limiting, server-side cache
with coalescing, bbox quantization, Grafana observability. That buys us time
before we need to own the upstream.
**Superseded for the Planner's POI overlays** by the
[`poi-index`](../../../openspec/changes/poi-index/) change (shipped), which
removes Overpass from the POI path entirely: instead of running the Overpass
software, a monthly osmium extract feeds a self-hosted PostGIS `planner.pois`
index served at `/api/pois`. That is lighter than this plan (no Overpass
engine, no diff replication, no RAM ceiling) and covers every current POI need
— all queries were simple tag-in-bbox lookups that never used Overpass QL.
The former interim proxy (`/api/overpass` → `overpass.private.coffee`) has been
**removed**.
**Still relevant if** we ever need genuine Overpass QL (arbitrary ad-hoc
queries, recursion, `around:`, set operations) that a precomputed category
index can't answer — that's the remaining reason to revive this. The Journal's
surface-breakdown backfill still queries public Overpass instances for way
geometry; owning that upstream is the other revive trigger.
## When to revive

2
docs/roadmap.md
View file

@ -83,7 +83,7 @@ See `docs/ideas/` for pre-spec explorations:
- `mobile-activity-recording/` — GPS tracking and activity logging from a native app
- `mobile-nearby-sync/` — BLE-based proximity discovery
- `self-host-overpass/` — Self-hosted Overpass API for POI overlays
- `self-host-overpass/` — Self-hosted Overpass API (superseded for POI overlays by the shipped `poi-index`; revive only if we need Overpass QL or to own the surface-backfill upstream)
- `instance-administration.md` — admin role, registration toggle, suspend/ban, federation blocklists, reports
- `social-interactions.md` — local likes + comments (the foundation the federated kudos/comments attach to)
- `activity-participants.md` — tag co-riders on shared activities, confirm/decline, federated mentions

9
e2e/fixtures/test.ts
View file

@ -8,10 +8,11 @@ import { test as base, expect } from "@playwright/test";
*
* The trigger for this safety net was #282 — a POI test that silently
* relied on live Overpass data for months because its `page.route`
* was pointing at a URL the browser no longer hit after `/api/overpass`
* was introduced. With this fixture in place, a request to
* `overpass.private.coffee` (or any other unlisted host) would abort
* and the test would surface the missing mock immediately.
* was pointing at a URL the browser no longer hit after the POI proxy
* was introduced. With this fixture in place, a request to any unlisted
* host would abort and the test would surface the missing mock
* immediately. (POIs are now served from the same-origin `/api/pois`
* index endpoint, so the map makes no third-party POI requests at all.)
*/
const EXTERNAL_ALLOWLIST: RegExp[] = [
// App origins served by Playwright's webServer

15
e2e/integration.test.ts
View file

@ -328,14 +328,15 @@ test.describe("Integration: BRouter routing", () => {
expect(response.status()).toBe(401);
});
test("rejects /api/overpass without an X-Trails-Session header", async ({ request }) => {
const response = await request.post(`${PLANNER}/api/overpass`, {
data: "data=[out:json];node[amenity=drinking_water](52.52,13.4,52.53,13.41);out;",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
Origin: PLANNER,
test("rejects /api/pois without an X-Trails-Session header", async ({ request }) => {
const response = await request.get(
`${PLANNER}/api/pois?bbox=52.52,13.40,52.53,13.41&categories=drinking_water`,
{
headers: {
Origin: PLANNER,
},
},
});
);
expect(response.status()).toBe(401);
});
});

7
e2e/planner-overlays.test.ts
View file

@ -30,17 +30,18 @@ test.describe("Planner – overlays", () => {
test("enable POI category shows markers on map", async ({ page, request }) => {
const url = await createSession(request);
await page.route("**/api/overpass", async (route) => {
await page.route("**/api/pois*", async (route) => {
await route.fulfill({
status: 200,
contentType: "application/json",
body: JSON.stringify({
elements: [
pois: [
{
type: "node",
id: 12345,
lat: 52.52,
lon: 13.405,
name: "Test Brunnen",
category: "drinking_water",
tags: { amenity: "drinking_water", name: "Test Brunnen" },
},
],

7
e2e/planner-routing.test.ts
View file

@ -108,17 +108,18 @@ test.describe("Planner – routing", () => {
test("nearby POI snap moves waypoint and prepends note prefix", async ({ page, request }) => {
const url = await createSession(request);
await page.route("**/api/overpass", async (route) => {
await page.route("**/api/pois*", async (route) => {
await route.fulfill({
status: 200,
contentType: "application/json",
body: JSON.stringify({
elements: [
pois: [
{
type: "node",
id: 99001,
lat: 52.521,
lon: 13.406,
name: "Stadtbrunnen",
category: "drinking_water",
tags: { amenity: "drinking_water", name: "Stadtbrunnen" },
},
],

8
infrastructure/.env.example
View file

@ -11,9 +11,11 @@ SESSION_SECRET=change-me
S3_ACCESS_KEY=
S3_SECRET_KEY=
# Overpass upstream failover. Comma-separated list tried in order;
# first 2xx response wins. Falls back to the single-URL OVERPASS_URL
# if this is unset, and to a built-in default list if neither is set.
# Overpass upstream failover for the Journal's surface-breakdown backfill
# (route-surface-breakdown). Comma-separated list tried in order; first 2xx
# response wins. Falls back to the single-URL OVERPASS_URL if unset, and to a
# built-in default list if neither is set. The Planner no longer uses Overpass
# — its POI overlays are served from the self-hosted `planner.pois` index.
# OVERPASS_URLS=https://lz4.overpass-api.de/api/interpreter,https://overpass-api.de/api/interpreter
# Demo-activity-bot. Only enable in prod. When true, the journal worker

22
infrastructure/brouter-host/Caddyfile
View file

@ -24,11 +24,23 @@
@authed header X-BRouter-Auth {$BROUTER_AUTH_TOKEN}
handle @authed {
reverse_proxy brouter:17777 {
# Don't forward the auth header upstream — BRouter doesn't
# use it, and it's cleaner to contain the credential at
# the proxy boundary.
header_up -X-BRouter-Auth
# POI index artifact + manifest published by poi-extract.sh. Served
# read-only from the mounted publish dir; the flagship import job
# pulls /poi/manifest.json and /poi/pois.ndjson.gz over the vSwitch.
# Same auth + vSwitch-only binding as BRouter, so it's unreachable
# from the public internet.
handle_path /poi/* {
root * /srv/poi
file_server
}
handle {
reverse_proxy brouter:17777 {
# Don't forward the auth header upstream — BRouter doesn't
# use it, and it's cleaner to contain the credential at
# the proxy boundary.
header_up -X-BRouter-Auth
}
}
}

3
infrastructure/brouter-host/docker-compose.yml
View file

@ -58,6 +58,9 @@ services:
BROUTER_AUTH_TOKEN: ${BROUTER_AUTH_TOKEN:?BROUTER_AUTH_TOKEN must be set}
volumes:
- ./Caddyfile:/etc/caddy/Caddyfile:ro
# POI index artifact published by poi-extract.sh, served read-only under
# /poi/* on the same vSwitch-only listener as BRouter.
- ./poi-extract/publish:/srv/poi:ro
- caddy-data:/data
- caddy-config:/config
networks:

121
infrastructure/brouter-host/poi-extract/README.md Normal file
View file

@ -0,0 +1,121 @@
# POI extract pipeline (BRouter host)
Monthly job that turns an OSM PBF into the compact POI artifact the flagship's
`planner.pois` index is built from. This is the "filter where the disk is"
half of the [`poi-index`](../../../openspec/changes/poi-index/) change — it
replaces the third-party Overpass dependency for the Planner's POI overlays.
Runs on the BRouter host as a non-root user (docker-group rights, no sudo). The
flagship import job (`infrastructure/scripts/poi-import.sh`) pulls the result.
## Files
| File | Role |
|------|------|
| `poi-extract.sh` | Download PBF → `osmium tags-filter` → `osmium export` → `to-ndjson.py` → gzip + manifest |
| `to-ndjson.py` | GeoJSONSeq → NDJSON, reducing ways/relations to their bbox centre (Overpass `out center` equivalent) |
| `osmium-filters.txt` | osmium tag filters, **generated** from `@trails-cool/map-core` POI selectors |
| `gen-osmium-filters.ts` | Regenerates `osmium-filters.txt` (run after changing categories) |
| `poi-extract.service` / `.timer` | systemd units (monthly) |
## Single source of truth
`osmium-filters.txt` is derived from `poiCategories` in
`packages/map-core/src/poi.ts`. After changing categories, regenerate it:
```bash
npx tsx infrastructure/brouter-host/poi-extract/gen-osmium-filters.ts
```
`packages/map-core/src/osmium-filters.sync.test.ts` fails CI if the committed
file drifts from the selectors, so the host never needs a Node runtime.
## Artifact contract
`poi-extract.sh` writes to the publish dir (default `./publish`, mounted into
the Caddy sidecar at `/srv/poi`):
- `pois.ndjson.gz` — one POI per line:
`{osm_type, osm_id, name, lat, lon, tags}`. Category is **not** stored here;
the flagship importer classifies each element from its `tags` using the same
map-core selectors (so the selector logic is never duplicated on this host).
- `manifest.json` — `{artifact, sha256, row_count, generated_at, source}`.
Caddy serves both under `/poi/*` on the existing BRouter listener, gated by the
same `X-BRouter-Auth` shared secret — so the artifact is reachable only by the
importer, not the public internet.
## Prerequisites
- **Docker** — osmium runs in a container (`osmium.Dockerfile`, built on first
use as `trails-osmium:local`). The pipeline's user is non-root with no sudo,
so osmium can't be apt-installed; docker-group rights cover it. On Linux this
adds no meaningful overhead — big files are bind-mounted, so I/O is native.
- **On the host** (already present): `python3`, `curl`, `gzip`, `sha256sum`.
## Install the timer
Deployed under the pipeline user's `~/brouter/poi-extract/`. As a user unit with
lingering (so the timer runs even when nobody is logged in):
```bash
loginctl enable-linger "$USER"
mkdir -p ~/.config/systemd/user
cp poi-extract.service poi-extract.timer ~/.config/systemd/user/
systemctl --user daemon-reload
systemctl --user enable --now poi-extract.timer
systemctl --user list-timers poi-extract.timer
```
Trigger a run by hand (e.g. the first bootstrap): `systemctl --user start
poi-extract.service` then `journalctl --user -u poi-extract.service -f`.
## Footprint
- **Bandwidth**: the planet PBF is ~80 GB downloaded per run (monthly). Point
`POI_PBF_URL` at a Geofabrik mirror or a regional/continental extract to cut
this — e.g. `europe-latest.osm.pbf` (~30 GB) or `germany-latest.osm.pbf`
(~4 GB) for a smaller instance.
- **Disk**: the planet PBF + filtered PBF live transiently in `./work` and are
removed on exit (even on failure) via a trap, so peak usage ≈ the PBF size
(tens of GB for planet). The published artifact is a few hundred MB gzipped.
Check the host has room for one PBF before a planet run (`df -h ~`).
- **CPU/time**: a full planet filter is IO-bound and can take a few hours; the
unit runs at `Nice=10` / idle IO priority so it doesn't disturb BRouter.
## Self-hosting
The pipeline is **optional**. A fresh instance works with an empty or absent
`planner.pois` table: `/api/pois` returns an empty result / 503 and the map's
POI panel shows "unavailable" while every other overlay and tile keeps working
— nothing breaks.
When you do want POIs, you don't need a second host or the planet. Both halves
run on one box, and `POI_PBF_URL` can point at any Geofabrik extract sized to
your region:
```bash
# Extract just your region (much smaller download, seconds not hours):
POI_PBF_URL=https://download.geofabrik.de/europe/germany-latest.osm.pbf \
./poi-extract.sh
# Then import it (first time: --bootstrap, since the live table is empty):
POI_ARTIFACT_BASE_URL=file-or-vswitch-url \
../../scripts/poi-import.sh --bootstrap
```
On a single-host instance, publish the artifact however is convenient (local
path, a static file server) and set `POI_ARTIFACT_BASE_URL` accordingly; the
two-host split is a flagship detail, not a requirement.
## Dry run first
Before the first planet run, validate the whole chain on a small extract:
```bash
POI_PBF_URL=https://download.geofabrik.de/europe/germany-latest.osm.pbf \
./poi-extract.sh
cat publish/manifest.json # row_count sane? sha256 present?
zcat publish/pois.ndjson.gz | head # records well-formed?
zcat publish/pois.ndjson.gz | wc -l
```

18
infrastructure/brouter-host/poi-extract/gen-osmium-filters.ts Normal file
View file

@ -0,0 +1,18 @@
/**
* Regenerate `osmium-filters.txt` from the POI category selectors in
* `@trails-cool/map-core` — the single source of truth for "what is a
* shelter". Run this after changing `poiCategories`:
*
* pnpm --filter @trails-cool/map-core exec tsx \
* ../../infrastructure/brouter-host/poi-extract/gen-osmium-filters.ts
*
* The committed file is what the (Node-free) BRouter host reads at extract
* time. `osmium-filters.sync.test.ts` in map-core fails CI if the two drift.
*/
import { writeFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { osmiumTagFilters } from "../../../packages/map-core/src/poi.ts";
const out = fileURLToPath(new URL("./osmium-filters.txt", import.meta.url));
writeFileSync(out, osmiumTagFilters().join("\n") + "\n");
console.log(`Wrote ${out}`);

3
infrastructure/brouter-host/poi-extract/osmium-filters.txt Normal file
View file

@ -0,0 +1,3 @@
nwr/amenity=drinking_water,water_point,shelter,restaurant,cafe,fast_food,pub,biergarten,bicycle_parking,bicycle_repair_station,bicycle_rental,toilets
nwr/tourism=wilderness_hut,camp_site,caravan_site,hotel,hostel,guest_house,viewpoint
nwr/shop=supermarket,convenience,bakery

15
infrastructure/brouter-host/poi-extract/osmium.Dockerfile Normal file
View file

@ -0,0 +1,15 @@
# Minimal osmium-tool image for the POI extract pipeline. The BRouter host runs
# a non-root `trails` user with no sudo, so osmium can't be apt-installed on the
# host — poi-extract.sh runs it in this container instead (docker-group rights
# are available). Big files are bind-mounted, so I/O is native; the container
# adds no meaningful CPU/throughput overhead on Linux.
#
# Built on first use by poi-extract.sh (tag: trails-osmium:local). Self-hosters
# get identical behavior without touching system packages.
FROM debian:bookworm-slim
RUN apt-get update \
&& apt-get install -y --no-install-recommends osmium-tool \
&& rm -rf /var/lib/apt/lists/*
ENTRYPOINT ["osmium"]

23
infrastructure/brouter-host/poi-extract/poi-extract.service Normal file
View file

@ -0,0 +1,23 @@
# POI extract — one-shot unit driven by poi-extract.timer. Runs as the
# non-root pipeline user (docker-group rights, no sudo). Install as a user unit:
# `systemctl --user enable --now poi-extract.timer` with lingering enabled
# (`loginctl enable-linger "$USER"`), or as a system unit with User=<that user>.
# See README.md.
[Unit]
Description=Filter an OSM PBF to trails.cool POI categories and publish the artifact
Wants=network-online.target
After=network-online.target
[Service]
Type=oneshot
# %h resolves to the running user's home; adjust if deployed elsewhere.
WorkingDirectory=%h/brouter/poi-extract
ExecStart=%h/brouter/poi-extract/poi-extract.sh
# The planet download + filter is long-running and IO-heavy; give it room and
# keep it off the box's foreground priority.
Nice=10
IOSchedulingClass=idle
TimeoutStartSec=6h
[Install]
WantedBy=default.target

128
infrastructure/brouter-host/poi-extract/poi-extract.sh Executable file
View file

@ -0,0 +1,128 @@
#!/bin/bash
# POI extract pipeline — runs on the dedicated BRouter host under the `trails`
# user. Downloads an OSM PBF, filters it to the planner's POI category selectors
# with osmium, reduces every element to a centroid, and publishes a compact
# NDJSON artifact + manifest that the flagship import job pulls over the vSwitch.
#
# Nothing here is planner/journal code — the host needs no Node runtime. The tag
# filter comes from `osmium-filters.txt`, generated from `@trails-cool/map-core`
# POI selectors (single source of truth; kept in sync by
# osmium-filters.sync.test.ts).
#
# osmium runs in a container: the `trails` user is non-root with no sudo, so
# osmium-tool can't be apt-installed on the host, but docker-group rights are
# available. Big files are bind-mounted into the container, so I/O is native and
# the container adds no meaningful CPU overhead on Linux. The image is built on
# first use from osmium.Dockerfile. python3/curl/gzip/sha256sum run on the host.
#
# Usage:
# ./poi-extract.sh
#
# Env:
# POI_PBF_URL OSM PBF to filter. Default: the full planet. A self-hoster or
# a dry run can point this at a Geofabrik regional extract, e.g.
# https://download.geofabrik.de/europe/germany-latest.osm.pbf
# POI_WORK_DIR Scratch dir for the (large, transient) planet download.
# Default: <script dir>/work
# POI_PUBLISH_DIR Where the artifact + manifest are written for Caddy to
# serve. Default: <script dir>/publish (mounted into the Caddy
# sidecar at /srv/poi — see docker-compose.yml + Caddyfile).
# OSMIUM_IMAGE osmium container image. Default: trails-osmium:local (built
# on first use from osmium.Dockerfile).
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
FILTERS_FILE="$SCRIPT_DIR/osmium-filters.txt"
TRANSFORMER="$SCRIPT_DIR/to-ndjson.py"
POI_PBF_URL="${POI_PBF_URL:-https://planet.osm.org/pbf/planet-latest.osm.pbf}"
POI_WORK_DIR="${POI_WORK_DIR:-$SCRIPT_DIR/work}"
POI_PUBLISH_DIR="${POI_PUBLISH_DIR:-$SCRIPT_DIR/publish}"
OSMIUM_IMAGE="${OSMIUM_IMAGE:-trails-osmium:local}"
PLANET_PBF="$POI_WORK_DIR/planet.osm.pbf"
FILTERED_PBF="$POI_WORK_DIR/filtered.osm.pbf"
ARTIFACT="$POI_PUBLISH_DIR/pois.ndjson.gz"
MANIFEST="$POI_PUBLISH_DIR/manifest.json"
mkdir -p "$POI_WORK_DIR" "$POI_PUBLISH_DIR"
# Run osmium inside the container with the work dir bind-mounted at /data, as the
# host uid/gid so output files are owned by `trails`, not root. Big-file I/O goes
# through the bind mount (native), never the container's overlay layer.
osmium_run() {
docker run --rm \
-u "$(id -u):$(id -g)" \
-v "$POI_WORK_DIR:/data" \
"$OSMIUM_IMAGE" "$@"
}
# Working files can be huge (planet PBF ~80 GB); always clean them up, even on
# failure, so a broken run can't fill the shared host's disk.
cleanup() { rm -f "$PLANET_PBF" "$FILTERED_PBF"; }
trap cleanup EXIT
log() { echo "[poi-extract] $*"; }
# --- 1. Read the tag filters (single source of truth: map-core) -------------
if [ ! -s "$FILTERS_FILE" ]; then
echo "ERROR: $FILTERS_FILE missing/empty — regenerate with gen-osmium-filters.ts" >&2
exit 1
fi
mapfile -t FILTERS < "$FILTERS_FILE"
log "tag filters: ${FILTERS[*]}"
# --- 2. Ensure the osmium image exists (build once from osmium.Dockerfile) ---
if ! docker image inspect "$OSMIUM_IMAGE" >/dev/null 2>&1; then
log "building osmium image $OSMIUM_IMAGE"
docker build -t "$OSMIUM_IMAGE" -f "$SCRIPT_DIR/osmium.Dockerfile" "$SCRIPT_DIR"
fi
# --- 3. Download the PBF -----------------------------------------------------
log "downloading $POI_PBF_URL"
curl --fail --location --show-error --silent --output "$PLANET_PBF" "$POI_PBF_URL"
log "downloaded $(du -h "$PLANET_PBF" | cut -f1)"
# --- 4. Filter to our categories --------------------------------------------
# Paths are inside the container's /data bind mount. tags-filter keeps the nodes
# referenced by matched ways/relations (default), so export can build geometry.
log "filtering with osmium tags-filter"
osmium_run tags-filter --overwrite --output /data/filtered.osm.pbf /data/planet.osm.pbf "${FILTERS[@]}"
log "filtered to $(du -h "$FILTERED_PBF" | cut -f1)"
# --- 5. Export to centroid NDJSON -------------------------------------------
# osmium export emits one GeoJSON feature per line to stdout (forwarded out of
# the container); to-ndjson.py reduces each to a bbox centre + compact record on
# the host and reports the row count on stderr.
log "exporting + transforming to NDJSON"
rowcount_file="$(mktemp)"
osmium_run export /data/filtered.osm.pbf \
--output-format=geojsonseq \
--add-unique-id=type_id \
--geometry-types=point,linestring,polygon \
--output - \
| python3 "$TRANSFORMER" 2>"$rowcount_file" \
| gzip -c > "$ARTIFACT"
ROW_COUNT="$(cat "$rowcount_file")"
rm -f "$rowcount_file"
log "wrote $ROW_COUNT rows -> $ARTIFACT ($(du -h "$ARTIFACT" | cut -f1))"
if [ "$ROW_COUNT" -eq 0 ]; then
echo "ERROR: 0 rows produced — refusing to publish an empty artifact" >&2
exit 1
fi
# --- 6. Manifest (checksum + timestamp + row count) -------------------------
SHA256="$(sha256sum "$ARTIFACT" | cut -d' ' -f1)"
GENERATED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
cat > "$MANIFEST" <<EOF
{
"artifact": "pois.ndjson.gz",
"sha256": "$SHA256",
"row_count": $ROW_COUNT,
"generated_at": "$GENERATED_AT",
"source": "$POI_PBF_URL"
}
EOF
log "manifest written: $MANIFEST"
log "done"

14
infrastructure/brouter-host/poi-extract/poi-extract.timer Normal file
View file

@ -0,0 +1,14 @@
# Monthly POI extract. POIs of these categories churn slowly, so monthly data
# is plenty (Organic Maps ships roughly monthly too). Runs early on the 1st;
# the flagship import timer is offset a day later to pull the fresh artifact.
[Unit]
Description=Monthly trails.cool POI extract
[Timer]
OnCalendar=*-*-01 02:00:00
# If the box was down at the scheduled time, run once it's back.
Persistent=true
RandomizedDelaySec=1h
[Install]
WantedBy=timers.target

93
infrastructure/brouter-host/poi-extract/to-ndjson.py Executable file
View file

@ -0,0 +1,93 @@
#!/usr/bin/env python3
"""Transform osmium's GeoJSONSeq output into the compact POI artifact.
Reads a GeoJSON Feature per line on stdin (as produced by
`osmium export -f geojsonseq --add-unique-id=type_id`) and writes one NDJSON
object per line on stdout:
{"osm_type": "n", "osm_id": "123", "name": "Brunnen",
"lat": 52.52, "lon": 13.40, "tags": {"amenity": "drinking_water", ...}}
Ways and relations are reduced to the centre of their bounding box — the same
representative point Overpass returned for `out center`, so markers land where
they used to. Category classification is intentionally NOT done here: the
flagship importer derives categories from the tags using the map-core
selectors (the single source of truth), so this host stays Node-free and the
selector logic is never duplicated.
"""
import json
import sys
def coords(geometry):
"""Yield every (lon, lat) pair in a GeoJSON geometry, any type."""
def walk(node):
if (
isinstance(node, list)
and len(node) == 2
and all(isinstance(n, (int, float)) for n in node)
):
yield node
elif isinstance(node, list):
for child in node:
yield from walk(child)
yield from walk(geometry.get("coordinates", []))
def bbox_center(geometry):
"""Bounding-box centre of a geometry, matching Overpass `out center`."""
pts = list(coords(geometry))
if not pts:
return None
lons = [p[0] for p in pts]
lats = [p[1] for p in pts]
return (min(lons) + max(lons)) / 2, (min(lats) + max(lats)) / 2
def main():
written = 0
for line in sys.stdin:
line = line.strip()
if not line:
continue
feature = json.loads(line)
geometry = feature.get("geometry") or {}
center = bbox_center(geometry)
if center is None:
continue
lon, lat = center
props = feature.get("properties") or {}
# osmium `--add-unique-id=type_id` emits ids like "n123", "w456",
# "r789" under the `id` / `@id` key depending on version.
raw_id = str(feature.get("id") or props.get("@id") or props.get("id") or "")
if not raw_id or raw_id[0] not in "nwr":
continue
osm_type, osm_id = raw_id[0], raw_id[1:]
# Tags are the feature properties minus osmium's synthetic id keys.
tags = {k: v for k, v in props.items() if k not in ("@id", "id")}
json.dump(
{
"osm_type": osm_type,
"osm_id": osm_id,
"name": tags.get("name"),
"lat": round(lat, 7),
"lon": round(lon, 7),
"tags": tags,
},
sys.stdout,
ensure_ascii=False,
separators=(",", ":"),
)
sys.stdout.write("\n")
written += 1
# Row count to stderr so the caller can record it in the manifest.
print(written, file=sys.stderr)
if __name__ == "__main__":
main()

1
infrastructure/docker-compose.staging.yml
View file

@ -94,7 +94,6 @@ services:
environment:
BROUTER_URL: ${BROUTER_URL:?BROUTER_URL must be set}
BROUTER_AUTH_TOKEN: ${BROUTER_AUTH_TOKEN:?BROUTER_AUTH_TOKEN must be set}
OVERPASS_URLS: ${OVERPASS_URLS:-https://lz4.overpass-api.de/api/interpreter,https://overpass-api.de/api/interpreter}
DATABASE_URL: postgres://trails:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in SOPS secrets.app.env}@postgres:5432/${STAGING_DATABASE}
NODE_ENV: production
PORT: 3001

6
infrastructure/docker-compose.yml
View file

@ -103,12 +103,6 @@ services:
# BRouter request. The Caddy sidecar on the dedicated host
# enforces this header. Set in SOPS secrets.app.env.
BROUTER_AUTH_TOKEN: ${BROUTER_AUTH_TOKEN:?BROUTER_AUTH_TOKEN must be set in SOPS secrets.app.env}
# Ordered failover list for the Overpass proxy. The code defaults
# to the same pair if unset; declaring it here makes the prod
# upstream explicit and easy to reshuffle via env when a
# community instance misbehaves. Override per-env with
# OVERPASS_URLS in the SOPS file.
OVERPASS_URLS: ${OVERPASS_URLS:-https://lz4.overpass-api.de/api/interpreter,https://overpass-api.de/api/interpreter}
DATABASE_URL: postgres://trails:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in SOPS secrets.app.env}@postgres:5432/trails
NODE_ENV: production
PORT: 3001

199
infrastructure/grafana/dashboards/planner.json
View file

@ -99,7 +99,7 @@
]
},
{
"title": "Overpass Upstream Health (5m success rate)",
"title": "POI Index Age",
"type": "stat",
"gridPos": {
"h": 5,
@ -109,8 +109,83 @@
},
"targets": [
{
"expr": "(sum(rate(overpass_upstream_requests_total{status=~\"2..\"}[5m])) or vector(0)) / sum(rate(overpass_upstream_requests_total[5m]))",
"legendFormat": "success"
"expr": "poi_index_age_seconds",
"legendFormat": "age"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "orange", "value": 3024000 },
{ "color": "red", "value": 3628800 }
]
}
}
}
},
{
"title": "POI Index Rows (total)",
"type": "stat",
"gridPos": {
"h": 5,
"w": 6,
"x": 6,
"y": 14
},
"targets": [
{
"expr": "sum(poi_index_rows)",
"legendFormat": "rows"
}
]
},
{
"title": "Last Import Status",
"type": "stat",
"gridPos": {
"h": 5,
"w": 6,
"x": 12,
"y": 14
},
"targets": [
{
"expr": "poi_import_last_status",
"legendFormat": "status"
}
],
"fieldConfig": {
"defaults": {
"mappings": [
{ "type": "value", "options": { "0": { "text": "FAIL", "color": "red" }, "1": { "text": "OK", "color": "green" } } }
],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "green", "value": 1 }
]
}
}
}
},
{
"title": "POI API Error Rate (5m)",
"type": "stat",
"gridPos": {
"h": 5,
"w": 6,
"x": 18,
"y": 14
},
"targets": [
{
"expr": "(sum(rate(poi_api_requests_total{status=~\"rate_limited|error\"}[5m])) or vector(0)) / clamp_min(sum(rate(poi_api_requests_total[5m])), 0.001)",
"legendFormat": "error ratio"
}
],
"fieldConfig": {
@ -121,76 +196,16 @@
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "orange", "value": 0.9 },
{ "color": "green", "value": 0.99 }
{ "color": "green", "value": null },
{ "color": "orange", "value": 0.05 },
{ "color": "red", "value": 0.2 }
]
}
}
}
},
{
"title": "Overpass Cache Hit Ratio (5m)",
"type": "stat",
"gridPos": {
"h": 5,
"w": 6,
"x": 6,
"y": 14
},
"targets": [
{
"expr": "(sum(rate(overpass_cache_events_total{result=\"hit\"}[5m])) or vector(0)) / sum(rate(overpass_cache_events_total[5m]))",
"legendFormat": "hit ratio"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"max": 1
}
}
},
{
"title": "Overpass Cache Size",
"type": "stat",
"gridPos": {
"h": 5,
"w": 6,
"x": 12,
"y": 14
},
"targets": [
{
"expr": "overpass_cache_size",
"legendFormat": "entries"
}
]
},
{
"title": "Overpass Upstream p95",
"type": "stat",
"gridPos": {
"h": 5,
"w": 6,
"x": 18,
"y": 14
},
"targets": [
{
"expr": "histogram_quantile(0.95, sum(rate(overpass_upstream_duration_seconds_bucket[5m])) by (le))",
"legendFormat": "p95"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
}
}
},
{
"title": "Overpass Cache Events",
"title": "POI API Requests (5m by status)",
"type": "timeseries",
"gridPos": {
"h": 7,
@ -200,28 +215,7 @@
},
"targets": [
{
"expr": "sum by (result) (rate(overpass_cache_events_total[5m]))",
"legendFormat": "{{result}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops"
}
}
},
{
"title": "Overpass Upstream Status",
"type": "timeseries",
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 19
},
"targets": [
{
"expr": "sum by (status) (rate(overpass_upstream_requests_total[5m]))",
"expr": "sum by (status) (rate(poi_api_requests_total[5m]))",
"legendFormat": "{{status}}"
}
],
@ -232,33 +226,20 @@
}
},
{
"title": "Overpass Upstream Latency",
"title": "POI Index Rows by Category",
"type": "timeseries",
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 26
"w": 12,
"x": 12,
"y": 19
},
"targets": [
{
"expr": "histogram_quantile(0.50, sum(rate(overpass_upstream_duration_seconds_bucket[5m])) by (le))",
"legendFormat": "p50"
},
{
"expr": "histogram_quantile(0.95, sum(rate(overpass_upstream_duration_seconds_bucket[5m])) by (le))",
"legendFormat": "p95"
},
{
"expr": "histogram_quantile(0.99, sum(rate(overpass_upstream_duration_seconds_bucket[5m])) by (le))",
"legendFormat": "p99"
"expr": "poi_index_rows",
"legendFormat": "{{category}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
}
}
]
}
]
}

29
infrastructure/grafana/provisioning/alerting/alerts.yml
View file

@ -265,16 +265,16 @@ groups:
annotations:
summary: "Prometheus TSDB compaction or WAL is failing — metric retention is at risk. Check the WAL for a corrupt segment (this silently capped retention to ~2h before)."
# Overpass upstream health. The public Overpass servers the Planner
# proxies to (overpass-api.de / lz4) flake — rate-limiting (429) and
# stalls. This alerts on a *sustained* upstream failure rate (>20%
# over 10m), distinct from the Caddy-502 alert which only sees the
# symptom. Brief blips (e.g. the ~30s stall on 2026-06-08) won't
# page; sustained degradation will. client-abort is excluded (that's
# the caller giving up, not the upstream failing). Per-status detail
# lives on the Planner dashboard.
- uid: overpass-upstream-unhealthy
title: Overpass upstream failure rate high
# POI index freshness. POIs are served from the self-hosted
# `planner.pois` index, refreshed by a monthly extract+import. If the
# index age exceeds ~6 weeks the monthly refresh has missed a cycle
# (extract job, vSwitch transfer, checksum, guard, or import failed) and
# the data is going stale. `poi_index_age_seconds` is published by the
# Planner from its own DB; a growing value also covers a silently-failing
# import (the guard refuses to swap, so age keeps climbing). Import-run
# detail lives on the Planner dashboard (poi_import_last_status).
- uid: poi-index-stale
title: POI index is stale
condition: B
noDataState: OK
data:
@ -282,7 +282,7 @@ groups:
relativeTimeRange: { from: 600, to: 0 }
datasourceUid: prometheus
model:
expr: (sum(rate(overpass_upstream_requests_total{status!~"2..|client-abort"}[10m])) or vector(0)) / clamp_min(sum(rate(overpass_upstream_requests_total[10m])), 0.001) * 100
expr: max(poi_index_age_seconds)
instant: true
- refId: B
datasourceUid: __expr__
@ -290,12 +290,13 @@ groups:
type: threshold
expression: A
conditions:
- evaluator: { params: [20], type: gt }
# 6 weeks = 3628800s — one missed monthly refresh.
- evaluator: { params: [3628800], type: gt }
operator: { type: and }
reducer: { type: last }
for: 10m
for: 1h
annotations:
summary: "Overpass upstream failure rate above 20% over 10m — public Overpass servers are degraded or rate-limiting. See the Planner dashboard; weigh self-hosting Overpass."
summary: "POI index age exceeds 6 weeks — the monthly refresh has missed a cycle. Check the poi-extract timer on the BRouter host and poi-import on the flagship (Planner dashboard: Last Import Status)."
contactPoints:
# Single "default" contact point with multiple integrations: every alert

45
infrastructure/scripts/README.md Normal file
View file

@ -0,0 +1,45 @@
# Flagship operational scripts
Deployed to `/opt/trails-cool/scripts/` by `cd-infra.yml`.
## `backup-postgres.sh`
Daily `pg_dump` + retention sweep. Run via cron (see the header).
## POI index import (`poi-import.sh`)
The "load where the database is" half of the [`poi-index`](../../openspec/changes/poi-index/)
change. Pulls the artifact the BRouter host published, classifies each element
into `planner.pois` category rows, and swaps it in atomically.
- `poi-import.sh` — fetch (vSwitch) → checksum → load → classify → **guarded
atomic rename swap**. `--bootstrap` skips the 70% row-count guard for the
first import (when the live table is legitimately empty).
- `poi-selectors.sql` — **generated** from `@trails-cool/map-core` selectors;
the classifier JOINs on it. Regenerate with
`npx tsx infrastructure/scripts/gen-poi-selectors-sql.ts` (CI sync test:
`packages/map-core/src/poi-selectors-sql.sync.test.ts`).
- `poi-import.service` / `.timer` — monthly, offset a day after the extract.
### First-time setup on the flagship
```bash
# 1. Create the empty live table from the Drizzle schema (idempotent).
# (Run from a checkout with DATABASE_URL pointed at production, or via the
# normal db:push deploy path.)
pnpm db:push
# 2. Install the timer.
cp /opt/trails-cool/scripts/poi-import.{service,timer} /etc/systemd/system/
systemctl daemon-reload
systemctl enable --now poi-import.timer
# 3. Bootstrap the first import (guard disabled — live table is empty).
BROUTER_AUTH_TOKEN=$(grep BROUTER_AUTH_TOKEN /opt/trails-cool/.env | cut -d= -f2-) \
/opt/trails-cool/scripts/poi-import.sh --bootstrap
```
Prometheus picks up import outcome from node_exporter's textfile collector
(`poi_import_last_status`, `…_last_success_timestamp_seconds`, `…_last_rows`)
when `NODE_EXPORTER_TEXTFILE_DIR` is set (the unit sets it). Index size and age
are additionally exposed by the planner at `/metrics`
(`poi_index_rows{category}`, `poi_index_age_seconds`).

36
infrastructure/scripts/gen-poi-selectors-sql.ts Normal file
View file

@ -0,0 +1,36 @@
/**
* Regenerate `poi-selectors.sql` from the POI category selectors in
* `@trails-cool/map-core` — the single source of truth. The flagship import
* script (`poi-import.sh`, bash + psql, no Node) loads this to classify each
* extracted OSM element into one row per matching category, mirroring
* `matchingCategoryIds`.
*
* Run after changing `poiCategories`:
* npx tsx infrastructure/scripts/gen-poi-selectors-sql.ts
*
* `packages/map-core/src/poi-selectors-sql.sync.test.ts` fails CI on drift.
*/
import { writeFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { poiCategories } from "../../packages/map-core/src/poi.ts";
function sqlLiteral(s: string): string {
return `'${s.replace(/'/g, "''")}'`;
}
const rows = poiCategories.flatMap((cat) =>
cat.selectors.map((s) => ` (${sqlLiteral(s.key)}, ${sqlLiteral(s.value)}, ${sqlLiteral(cat.id)})`),
);
const sql = `-- GENERATED from @trails-cool/map-core poiCategories. Do not edit by hand;
-- regenerate with: npx tsx infrastructure/scripts/gen-poi-selectors-sql.ts
-- Loaded by poi-import.sh inside the import transaction to classify each
-- extracted OSM element into one row per matching category.
CREATE TEMP TABLE poi_selector (key text, value text, category text) ON COMMIT DROP;
INSERT INTO poi_selector (key, value, category) VALUES
${rows.join(",\n")};
`;
const out = fileURLToPath(new URL("./poi-selectors.sql", import.meta.url));
writeFileSync(out, sql);
console.log(`Wrote ${out}`);

19
infrastructure/scripts/poi-import.service Normal file
View file

@ -0,0 +1,19 @@
# POI import — one-shot unit driven by poi-import.timer. Runs on the flagship
# as root (the deploy user), pulling the artifact the BRouter host published and
# swapping it into planner.pois. Install: copy to /etc/systemd/system/, then
# `systemctl enable --now poi-import.timer`. See infrastructure/scripts/README
# / the poi-index change docs. Requires BROUTER_AUTH_TOKEN in the environment
# file below (reuse the deployed /opt/trails-cool/.env).
[Unit]
Description=Import the trails.cool POI index artifact into planner.pois
Wants=network-online.target
After=network-online.target docker.service
[Service]
Type=oneshot
WorkingDirectory=/opt/trails-cool
EnvironmentFile=/opt/trails-cool/.env
# Optional: expose import-outcome metrics to node_exporter's textfile collector.
Environment=NODE_EXPORTER_TEXTFILE_DIR=/var/lib/node_exporter/textfile_collector
ExecStart=/opt/trails-cool/scripts/poi-import.sh
TimeoutStartSec=2h

162
infrastructure/scripts/poi-import.sh Executable file
View file

@ -0,0 +1,162 @@
#!/bin/bash
# POI import — runs on the flagship. Pulls the POI artifact the BRouter host
# published, classifies each element into category rows, and swaps it into the
# live `planner.pois` table atomically. This is the "load where the database
# is" half of the poi-index change.
#
# Serving is never interrupted: the new data is built in a side table and the
# live table is replaced with a rename inside one transaction. A truncated
# download can never blank the map — the swap is refused if the fresh dataset
# has fewer than 70% of the live table's rows (override with --bootstrap on the
# very first import, when the live table is legitimately empty).
#
# Prerequisites: the live table must already exist (created empty by
# `pnpm db:push` from the Drizzle schema); curl, gunzip, sha256sum on the host.
#
# Usage:
# ./poi-import.sh [--bootstrap]
#
# Env:
# POI_ARTIFACT_BASE_URL vSwitch URL of the published artifact dir.
# Default: http://10.0.1.10:17777/poi
# BROUTER_AUTH_TOKEN Shared secret Caddy requires (X-BRouter-Auth).
# COMPOSE_FILE docker compose file. Default: /opt/trails-cool/docker-compose.yml
# NODE_EXPORTER_TEXTFILE_DIR If set, an import-outcome .prom metric is written here.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SELECTORS_SQL="$SCRIPT_DIR/poi-selectors.sql"
POI_ARTIFACT_BASE_URL="${POI_ARTIFACT_BASE_URL:-http://10.0.1.10:17777/poi}"
COMPOSE_FILE="${COMPOSE_FILE:-/opt/trails-cool/docker-compose.yml}"
BOOTSTRAP=0
[ "${1:-}" = "--bootstrap" ] && BOOTSTRAP=1
WORK_DIR="$(mktemp -d)"
MANIFEST="$WORK_DIR/manifest.json"
ARTIFACT="$WORK_DIR/pois.ndjson.gz"
cleanup() { rm -rf "$WORK_DIR"; }
trap cleanup EXIT
log() { echo "[poi-import] $*"; }
psql() { docker compose -f "$COMPOSE_FILE" exec -T postgres psql -U trails -d trails -v ON_ERROR_STOP=1 "$@"; }
emit_metric() {
# Best-effort node_exporter textfile metric. status=1 success, 0 failure.
local status="$1" rows="${2:-0}"
[ -n "${NODE_EXPORTER_TEXTFILE_DIR:-}" ] || return 0
local now; now="$(date +%s)"
local tmp; tmp="$(mktemp)"
{
echo "# HELP poi_import_last_run_timestamp_seconds Unix time of the last POI import run."
echo "# TYPE poi_import_last_run_timestamp_seconds gauge"
echo "poi_import_last_run_timestamp_seconds $now"
echo "# HELP poi_import_last_status 1 if the last POI import swapped successfully, else 0."
echo "# TYPE poi_import_last_status gauge"
echo "poi_import_last_status $status"
echo "# HELP poi_import_last_rows Row count of the last successful POI import."
echo "# TYPE poi_import_last_rows gauge"
echo "poi_import_last_rows $rows"
if [ "$status" = "1" ]; then
echo "# HELP poi_import_last_success_timestamp_seconds Unix time of the last successful POI import."
echo "# TYPE poi_import_last_success_timestamp_seconds gauge"
echo "poi_import_last_success_timestamp_seconds $now"
fi
} > "$tmp"
mv "$tmp" "$NODE_EXPORTER_TEXTFILE_DIR/poi_import.prom"
}
fail() { log "ERROR: $*"; emit_metric 0; exit 1; }
[ -s "$SELECTORS_SQL" ] || fail "$SELECTORS_SQL missing — regenerate with gen-poi-selectors-sql.ts"
AUTH_HEADER=()
[ -n "${BROUTER_AUTH_TOKEN:-}" ] && AUTH_HEADER=(-H "X-BRouter-Auth: ${BROUTER_AUTH_TOKEN}")
# --- 1. Fetch manifest + artifact over the vSwitch ---------------------------
log "fetching manifest from $POI_ARTIFACT_BASE_URL"
curl --fail --silent --show-error "${AUTH_HEADER[@]}" -o "$MANIFEST" "$POI_ARTIFACT_BASE_URL/manifest.json" \
|| fail "manifest fetch failed"
EXPECTED_SHA="$(grep -o '"sha256"[[:space:]]*:[[:space:]]*"[0-9a-f]*"' "$MANIFEST" | grep -o '[0-9a-f]\{64\}')"
MANIFEST_ROWS="$(grep -o '"row_count"[[:space:]]*:[[:space:]]*[0-9]*' "$MANIFEST" | grep -o '[0-9]*')"
[ -n "$EXPECTED_SHA" ] || fail "manifest has no sha256"
log "manifest: row_count=$MANIFEST_ROWS sha256=${EXPECTED_SHA:0:12}…"
log "fetching artifact"
curl --fail --silent --show-error "${AUTH_HEADER[@]}" -o "$ARTIFACT" "$POI_ARTIFACT_BASE_URL/pois.ndjson.gz" \
|| fail "artifact fetch failed"
# --- 2. Verify checksum ------------------------------------------------------
ACTUAL_SHA="$(sha256sum "$ARTIFACT" | cut -d' ' -f1)"
[ "$ACTUAL_SHA" = "$EXPECTED_SHA" ] || fail "checksum mismatch (got ${ACTUAL_SHA:0:12}…, want ${EXPECTED_SHA:0:12}…)"
log "checksum verified"
# --- 3. Load raw NDJSON into a staging import table --------------------------
log "loading raw NDJSON"
psql -c "CREATE TABLE IF NOT EXISTS planner.pois_raw_import (doc jsonb); TRUNCATE planner.pois_raw_import;"
# Each gzip line is one JSON object. Load it as a single column value using
# control-char delimiter/quote that never appear in JSON so the whole line
# lands intact, then it's cast to jsonb.
gunzip -c "$ARTIFACT" | psql -c \
"\copy planner.pois_raw_import (doc) FROM STDIN WITH (FORMAT csv, DELIMITER E'\x1f', QUOTE E'\x1e')" \
|| fail "COPY failed"
# --- 4. Build the staging table, classified into category rows ---------------
# One transaction: temp selector table (from map-core) → pois_staging built
# LIKE the live table so structure/defaults match the Drizzle schema exactly →
# classified insert → indexes. The 70% guard + swap run in step 5.
log "building classified staging table"
psql <<SQL || fail "staging build failed"
BEGIN;
\\i $SELECTORS_SQL
DROP TABLE IF EXISTS planner.pois_staging;
CREATE TABLE planner.pois_staging (LIKE planner.pois INCLUDING DEFAULTS);
INSERT INTO planner.pois_staging (osm_type, osm_id, category, name, geom, tags, imported_at)
SELECT r.doc->>'osm_type',
r.doc->>'osm_id',
s.category,
r.doc->>'name',
ST_SetSRID(ST_MakePoint((r.doc->>'lon')::float8, (r.doc->>'lat')::float8), 4326),
COALESCE(r.doc->'tags', '{}'::jsonb),
now()
FROM planner.pois_raw_import r
JOIN poi_selector s ON r.doc->'tags'->>s.key = s.value;
ALTER TABLE planner.pois_staging
ADD CONSTRAINT pois_staging_pkey PRIMARY KEY (osm_type, osm_id, category);
CREATE INDEX pois_staging_geom_idx ON planner.pois_staging USING gist (geom);
CREATE INDEX pois_staging_category_idx ON planner.pois_staging (category);
COMMIT;
SQL
STAGING_ROWS="$(psql -tA -c "SELECT count(*) FROM planner.pois_staging;")"
LIVE_ROWS="$(psql -tA -c "SELECT count(*) FROM planner.pois;")"
log "staging=$STAGING_ROWS live=$LIVE_ROWS bootstrap=$BOOTSTRAP"
[ "$STAGING_ROWS" -gt 0 ] || fail "staging is empty — refusing to swap"
# --- 5. 70% guard + atomic swap ---------------------------------------------
if [ "$BOOTSTRAP" -ne 1 ] && [ "$LIVE_ROWS" -gt 0 ]; then
# Integer 70% threshold: staging * 100 >= live * 70
if [ $(( STAGING_ROWS * 100 )) -lt $(( LIVE_ROWS * 70 )) ]; then
fail "guard tripped: staging ($STAGING_ROWS) < 70% of live ($LIVE_ROWS). No swap. Use --bootstrap to override."
fi
fi
log "swapping staging into live (atomic)"
psql <<'SQL' || fail "swap failed"
BEGIN;
DROP TABLE IF EXISTS planner.pois_old;
ALTER TABLE planner.pois RENAME TO pois_old;
ALTER TABLE planner.pois_staging RENAME TO pois;
-- Rename constraints/indexes to the canonical Drizzle names so a later
-- `db:push` sees no drift. pois_old still holds the old canonical names until
-- it's dropped, so free them first.
DROP TABLE planner.pois_old;
ALTER TABLE planner.pois RENAME CONSTRAINT pois_staging_pkey TO pois_pkey;
ALTER INDEX planner.pois_staging_geom_idx RENAME TO pois_geom_idx;
ALTER INDEX planner.pois_staging_category_idx RENAME TO pois_category_idx;
COMMIT;
SQL
psql -c "DROP TABLE IF EXISTS planner.pois_raw_import;"
log "done: live table now has $STAGING_ROWS rows"
emit_metric 1 "$STAGING_ROWS"

12
infrastructure/scripts/poi-import.timer Normal file
View file

@ -0,0 +1,12 @@
# Monthly POI import, offset a day after the BRouter host's extract (which runs
# on the 1st) so the fresh artifact is already published when this pulls it.
[Unit]
Description=Monthly trails.cool POI index import
[Timer]
OnCalendar=*-*-02 04:00:00
Persistent=true
RandomizedDelaySec=30m
[Install]
WantedBy=timers.target

28
infrastructure/scripts/poi-selectors.sql Normal file
View file

@ -0,0 +1,28 @@
-- GENERATED from @trails-cool/map-core poiCategories. Do not edit by hand;
-- regenerate with: npx tsx infrastructure/scripts/gen-poi-selectors-sql.ts
-- Loaded by poi-import.sh inside the import transaction to classify each
-- extracted OSM element into one row per matching category.
CREATE TEMP TABLE poi_selector (key text, value text, category text) ON COMMIT DROP;
INSERT INTO poi_selector (key, value, category) VALUES
('amenity', 'drinking_water', 'drinking_water'),
('amenity', 'water_point', 'drinking_water'),
('amenity', 'shelter', 'shelter'),
('tourism', 'wilderness_hut', 'shelter'),
('tourism', 'camp_site', 'camping'),
('tourism', 'caravan_site', 'camping'),
('amenity', 'restaurant', 'food'),
('amenity', 'cafe', 'food'),
('amenity', 'fast_food', 'food'),
('amenity', 'pub', 'food'),
('amenity', 'biergarten', 'food'),
('shop', 'supermarket', 'groceries'),
('shop', 'convenience', 'groceries'),
('shop', 'bakery', 'groceries'),
('amenity', 'bicycle_parking', 'bike_infra'),
('amenity', 'bicycle_repair_station', 'bike_infra'),
('amenity', 'bicycle_rental', 'bike_infra'),
('tourism', 'hotel', 'accommodation'),
('tourism', 'hostel', 'accommodation'),
('tourism', 'guest_house', 'accommodation'),
('tourism', 'viewpoint', 'viewpoints'),
('amenity', 'toilets', 'toilets');

50
openspec/changes/poi-index/tasks.md
View file

@ -1,51 +1,51 @@
## 1. Category selectors (single source of truth)
- [ ] 1.1 Refactor `packages/map-core/src/poi.ts`: replace Overpass QL `query` strings with structured `selectors: Array<{key, value}>` per category; update `poi.test.ts`
- [ ] 1.2 Add a helper that renders the selectors as an osmium tag-filter expression list (used by the pipeline; unit-tested against the nine categories)
- [x] 1.1 Refactor `packages/map-core/src/poi.ts`: replace Overpass QL `query` strings with structured `selectors: Array<{key, value}>` per category; update `poi.test.ts`
- [x] 1.2 Add a helper that renders the selectors as an osmium tag-filter expression list (used by the pipeline; unit-tested against the nine categories)
## 2. Schema
- [ ] 2.1 Add `planner.pois` to `packages/db` (osm_type, osm_id, category, name, geom Point 4326, tags jsonb, imported_at; PK (osm_type, osm_id, category); GiST on geom, btree on category) + migration
- [ ] 2.2 `pnpm db:push` locally and verify bbox+category query plans use the indexes
- [x] 2.1 Add `planner.pois` to `packages/db` (osm_type, osm_id, category, name, geom Point 4326, tags jsonb, imported_at; PK (osm_type, osm_id, category); GiST on geom, btree on category) + migration
- [x] 2.2 `pnpm db:push` locally and verify bbox+category query plans use the indexes _(verified: serving query drives off `pois_geom_idx` for the bbox; `pois_category_idx` used for category filter; PK created as `pois_pkey`)_
## 3. Extract pipeline (BRouter host)
- [ ] 3.1 Create `infrastructure/brouter-host/poi-extract/`: script that downloads the configured PBF (`POI_PBF_URL`, planet by default), runs `osmium tag-filter` + `osmium export` (centroids) into gzipped NDJSON `{osm_type, osm_id, categories, name, lat, lon, tags}`, writes a manifest (sha256 + timestamp + row count), and cleans up working files
- [ ] 3.2 Publish artifact + manifest via the existing Caddy sidecar at a vSwitch-only address; verify it is unreachable from the public internet
- [ ] 3.3 Add a monthly systemd timer under the `trails` user; document disk/bandwidth footprint in `infrastructure/brouter-host/poi-extract/README.md`
- [ ] 3.4 Dry-run on a small Geofabrik extract first; then a full planet run — record row counts per category and artifact size
- [x] 3.1 Create `infrastructure/brouter-host/poi-extract/`: script that downloads the configured PBF (`POI_PBF_URL`, planet by default), runs `osmium tags-filter` + `osmium export` (centroids) into gzipped NDJSON, writes a manifest (sha256 + timestamp + row count), and cleans up working files _(NDJSON is `{osm_type, osm_id, name, lat, lon, tags}`; the importer derives `categories` from tags via map-core, so selector logic isn't duplicated on the Node-free host)_
- [x] 3.2 Publish artifact + manifest via the existing Caddy sidecar at a vSwitch-only address (`/poi/*` on `:17777`, same `X-BRouter-Auth` gate) _(public-unreachability follows from the existing vSwitch-only bind + UFW; verify on the host)_
- [x] 3.3 Add a monthly systemd timer under the `trails` user; document disk/bandwidth footprint in `infrastructure/brouter-host/poi-extract/README.md`
- [ ] 3.4 Dry-run on a small Geofabrik extract first; then a full planet run — record row counts per category and artifact size _(operational: run on the BRouter host)_
## 4. Import job (flagship)
- [ ] 4.1 Create the import script: fetch manifest + artifact over the vSwitch, verify checksum, COPY into `planner.pois_staging`, build indexes, atomic rename swap in one transaction
- [ ] 4.2 Implement the 70% row-count guard with a `--bootstrap` override; abort loudly (log + metric) when tripped
- [ ] 4.3 Schedule via systemd timer offset from the extract job; wire into `cd-infra.yml` deploy sources (note: config-file-only changes need `--force-recreate` where applicable)
- [ ] 4.4 First production import with `--bootstrap`; spot-check several viewports against live Overpass results for category parity
- [x] 4.1 Create the import script: fetch manifest + artifact over the vSwitch, verify checksum, COPY into `planner.pois_staging`, build indexes, atomic rename swap in one transaction
- [x] 4.2 Implement the 70% row-count guard with a `--bootstrap` override; abort loudly (log + metric) when tripped
- [x] 4.3 Schedule via systemd timer (`poi-import.timer`, offset a day after the extract); wire into `cd-infra.yml` deploy sources (`infrastructure/scripts` added to the SCP list)
- [ ] 4.4 First production import with `--bootstrap`; spot-check several viewports against live Overpass results for category parity _(operational: run on the flagship)_
## 5. Serving route
- [ ] 5.1 Create `apps/planner/app/routes/api.pois.ts` (GET bbox + categories): same-origin + session checks, 120/IP/min rate limit (no DB query on rejection), 100-result cap, short Cache-Control; register in `apps/planner/app/routes.ts`
- [ ] 5.2 Unit tests: happy path, invalid bbox/categories 400, 429 path, empty-table graceful response
- [x] 5.1 Create `apps/planner/app/routes/api.pois.ts` (GET bbox + categories): same-origin + session checks, 120/IP/min rate limit (no DB query on rejection), 100-result cap, short Cache-Control; register in `apps/planner/app/routes.ts`
- [x] 5.2 Unit tests: happy path, invalid bbox/categories 400, 429 path, empty-table graceful response
## 6. Client switch
- [ ] 6.1 Rewrite `apps/planner/app/lib/overpass.ts` as the `/api/pois` client (keep `Poi` shape, `quantizeBbox`, error mapping; drop QL building); rename file to `pois.ts` and update imports (`use-pois`, `use-nearby-pois`, snap-to-poi)
- [ ] 6.2 Update client tests; verify the existing rate-limit banner and unavailable message fire on 429/5xx from `/api/pois`
- [ ] 6.3 Delete `apps/planner/app/routes/api.overpass.ts` and its tests; remove the route registration and the private.coffee configuration
- [x] 6.1 Rewrite `apps/planner/app/lib/overpass.ts` as the `/api/pois` client (keep `Poi` shape, `quantizeBbox`, error mapping; drop QL building); rename file to `pois.ts` and update imports (`use-pois`, `use-nearby-pois`, snap-to-poi)
- [x] 6.2 Update client tests; verify the existing rate-limit banner and unavailable message fire on 429/5xx from `/api/pois`
- [x] 6.3 Delete `apps/planner/app/routes/api.overpass.ts` and its tests; remove the route registration and the private.coffee configuration (`OVERPASS_URLS` kept for the Journal surface backfill; removed from the planner service in both compose files)
## 7. Observability
- [ ] 7.1 Add `poi_index_rows{category}`, `poi_index_age_seconds`, import outcome, and `poi_api_requests_total{status}` metrics; remove `overpass_upstream_*` metrics from `metrics.server.ts`
- [ ] 7.2 Update the Grafana planner dashboard: replace Overpass upstream panels with index freshness + serving panels; add the stale-index alert (~6 weeks)
- [x] 7.1 Add `poi_index_rows{category}`, `poi_index_age_seconds`, import outcome, and `poi_api_requests_total{status}` metrics; remove `overpass_upstream_*` metrics from `metrics.server.ts` (planner exposes rows/age via scrape-time DB collect + `poi_api_requests_total`; import outcome is emitted by the import job via node_exporter textfile — see Task 4)
- [x] 7.2 Update the Grafana planner dashboard: replace Overpass upstream panels with index freshness + serving panels; add the stale-index alert (~6 weeks) (`poi-index-stale` in alerts.yml)
## 8. Docs & cleanup
- [ ] 8.1 Update `docs/architecture.md` (POI data flow) and the privacy documentation (no third-party POI requests)
- [ ] 8.2 Document the self-hoster story: optional pipeline, regional extract URL, graceful behavior with an empty index
- [ ] 8.3 Update `docs/ideas/self-host-overpass/README.md`: superseded by `poi-index`, with the revive-criteria note kept for the QL-compatibility case
- [x] 8.1 Update `docs/architecture.md` (POI data flow) and the privacy documentation (Overpass now Journal-surface-backfill-only; Planner POIs served same-origin)
- [x] 8.2 Document the self-hoster story: optional pipeline, regional extract URL, graceful behavior with an empty index (poi-extract README)
- [x] 8.3 Update `docs/ideas/self-host-overpass/README.md`: superseded by `poi-index`, with the revive-criteria note kept for the QL-compatibility case (+ roadmap pointer)
## 9. Verification
- [ ] 9.1 E2E: POI overlay flow against a seeded local `planner.pois` table (enable category → markers appear; popup shows name/hours/OSM link)
- [ ] 9.2 Run `pnpm typecheck && pnpm lint && pnpm test && pnpm test:e2e`
- [ ] 9.3 Post-cutover production check: enable each of the nine categories on a busy viewport and a rural viewport; confirm results and dashboard metrics
- [x] 9.1 E2E: POI overlay flow (enable category → markers appear) — `e2e/planner-overlays.test.ts` + nearby-snap in `planner-routing.test.ts` now mock `/api/pois` (network-mock is the house pattern; a DB-seeded variant is possible but the route's SQL path is covered by `api.pois.test.ts`)
- [x] 9.2 Run `pnpm typecheck && pnpm lint && pnpm test && pnpm test:e2e` _(typecheck + lint + unit green; full planner e2e project green incl. all three POI tests. Journal e2e specs need `E2E=true` + the `trails_e2e` scratch DB — unrelated to this planner-only change; CI runs them with that harness.)_
- [ ] 9.3 Post-cutover production check: enable each of the nine categories on a busy viewport and a rural viewport; confirm results and dashboard metrics _(operational: after the first production import)_

40
packages/db/src/schema/planner.ts
View file

@ -1,4 +1,4 @@
import { pgSchema, text, timestamp, boolean, customType, index } from "drizzle-orm/pg-core";
import { pgSchema, text, timestamp, boolean, jsonb, customType, index, primaryKey } from "drizzle-orm/pg-core";
const bytea = customType<{ data: Buffer }>({
dataType() {
@ -6,6 +6,15 @@ const bytea = customType<{ data: Buffer }>({
},
});
// Centroid point in WGS84. Ways/relations are reduced to their centroid at
// extract time (matching the old Overpass `out center`), so every POI is a
// single point regardless of its OSM geometry.
const point = customType<{ data: string }>({
dataType() {
return "geometry(Point, 4326)";
},
});
export const plannerSchema = pgSchema("planner");
export const sessions = plannerSchema.table("sessions", {
@ -22,3 +31,32 @@ export const sessions = plannerSchema.table("sessions", {
// total sessions ever created.
lastActivityIdx: index("sessions_last_activity_idx").on(t.lastActivity),
}));
/**
* Self-hosted POI index. Fed monthly by the extract pipeline (osmium filter on
* the BRouter host) + the flagship import job's atomic staging swap — never
* written at request time. Served read-only by `/api/pois` for the planner's
* map overlays, replacing the former Overpass proxy.
*
* An OSM element that matches more than one category is stored once per
* category (part of the composite primary key), because the client treats each
* category's markers independently.
*/
export const pois = plannerSchema.table("pois", {
// OSM element type: 'n' (node), 'w' (way), or 'r' (relation).
osmType: text("osm_type").notNull(),
osmId: text("osm_id").notNull(),
category: text("category").notNull(),
name: text("name"),
geom: point("geom").notNull(),
tags: jsonb("tags").$type<Record<string, string>>().notNull(),
importedAt: timestamp("imported_at", { withTimezone: true }).notNull().defaultNow(),
}, (t) => ({
// Named explicitly so the import job's atomic swap can rename the staging
// table's PK to this exact name — keeping `db:push` drift-free post-swap.
pk: primaryKey({ name: "pois_pkey", columns: [t.osmType, t.osmId, t.category] }),
// GiST spatial index for the bbox intersection in the serving query.
geomIdx: index("pois_geom_idx").using("gist", t.geom),
// btree on category so `category = ANY($cats)` is index-driven.
categoryIdx: index("pois_category_idx").on(t.category),
}));

10
packages/map-core/src/index.ts
View file

@ -12,8 +12,14 @@ export {
maxspeedColor,
} from "./colors/index.ts";
export type { PoiCategory } from "./poi.ts";
export { poiCategories, getCategoriesForProfile, profileOverlayDefaults } from "./poi.ts";
export type { PoiCategory, PoiSelector } from "./poi.ts";
export {
poiCategories,
getCategoriesForProfile,
matchingCategoryIds,
osmiumTagFilters,
profileOverlayDefaults,
} from "./poi.ts";
export {
Z_CURSOR, Z_GHOST_WAYPOINT, Z_WAYPOINT,

22
packages/map-core/src/osmium-filters.sync.test.ts Normal file
View file

@ -0,0 +1,22 @@
import { describe, it, expect } from "vitest";
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { osmiumTagFilters } from "./poi.ts";
// The BRouter host reads the committed `osmium-filters.txt` at extract time —
// it has no Node runtime to derive filters from map-core. This test fails CI
// if the file drifts from the selectors, so `poiCategories` stays the single
// source of truth. Regenerate with:
// npx tsx infrastructure/brouter-host/poi-extract/gen-osmium-filters.ts
describe("osmium-filters.txt", () => {
it("matches the filters derived from poiCategories", () => {
const path = fileURLToPath(
new URL(
"../../../infrastructure/brouter-host/poi-extract/osmium-filters.txt",
import.meta.url,
),
);
const committed = readFileSync(path, "utf8").trimEnd();
expect(committed).toBe(osmiumTagFilters().join("\n"));
});
});

27
packages/map-core/src/poi-selectors-sql.sync.test.ts Normal file
View file

@ -0,0 +1,27 @@
import { describe, it, expect } from "vitest";
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { poiCategories } from "./poi.ts";
// The flagship import script (poi-import.sh, bash + psql, no Node) loads the
// committed poi-selectors.sql to classify extracted elements into category
// rows. This test fails CI if that file drifts from the map-core selectors, so
// poiCategories stays the single source of truth. Regenerate with:
// npx tsx infrastructure/scripts/gen-poi-selectors-sql.ts
describe("poi-selectors.sql", () => {
it("contains a VALUES row for every selector/category pair", () => {
const path = fileURLToPath(
new URL("../../../infrastructure/scripts/poi-selectors.sql", import.meta.url),
);
const sql = readFileSync(path, "utf8");
const expected = poiCategories.flatMap((cat) =>
cat.selectors.map((s) => `('${s.key}', '${s.value}', '${cat.id}')`),
);
for (const row of expected) {
expect(sql).toContain(row);
}
// No extra/stale rows: the count of VALUES tuples equals the selector count.
const tupleCount = (sql.match(/\(\s*'[^']+',\s*'[^']+',\s*'[^']+'\s*\)/g) ?? []).length;
expect(tupleCount).toBe(expected.length);
});
});

71
packages/map-core/src/poi.test.ts
View file

@ -1,5 +1,11 @@
import { describe, it, expect } from "vitest";
import { poiCategories, getCategoriesForProfile, profileOverlayDefaults } from "./poi.ts";
import {
poiCategories,
getCategoriesForProfile,
matchingCategoryIds,
osmiumTagFilters,
profileOverlayDefaults,
} from "./poi.ts";
describe("poiCategories", () => {
it("has expected categories", () => {
@ -16,7 +22,11 @@ describe("poiCategories", () => {
expect(cat.name).toBeTruthy();
expect(cat.icon).toBeTruthy();
expect(cat.color).toMatch(/^#[0-9a-fA-F]{6}$/);
expect(cat.query).toBeTruthy();
expect(cat.selectors.length).toBeGreaterThan(0);
for (const sel of cat.selectors) {
expect(sel.key).toBeTruthy();
expect(sel.value).toBeTruthy();
}
}
});
});
@ -45,6 +55,63 @@ describe("getCategoriesForProfile", () => {
});
});
describe("matchingCategoryIds", () => {
it("classifies an element by its tags", () => {
expect(matchingCategoryIds({ amenity: "drinking_water" })).toEqual(["drinking_water"]);
expect(matchingCategoryIds({ tourism: "viewpoint", name: "Aussicht" })).toEqual(["viewpoints"]);
});
it("returns an empty array for elements matching no category", () => {
expect(matchingCategoryIds({ highway: "primary" })).toEqual([]);
expect(matchingCategoryIds({})).toEqual([]);
});
it("returns every matching category when tags satisfy more than one", () => {
// A drinking water point that is also tagged as toilets belongs to both.
const ids = matchingCategoryIds({ amenity: "toilets", drinking_water: "yes" });
expect(ids).toContain("toilets");
});
it("matches secondary selectors within a category", () => {
expect(matchingCategoryIds({ amenity: "water_point" })).toEqual(["drinking_water"]);
expect(matchingCategoryIds({ tourism: "wilderness_hut" })).toEqual(["shelter"]);
});
});
describe("osmiumTagFilters", () => {
it("renders one nwr expression per distinct key, grouping values", () => {
const filters = osmiumTagFilters();
// amenity spans several categories; all its values collapse into one expr.
const amenity = filters.find((f) => f.startsWith("nwr/amenity="));
expect(amenity).toBeDefined();
expect(amenity).toContain("drinking_water");
expect(amenity).toContain("toilets");
expect(amenity).toContain("restaurant");
// tourism and shop each get their own expression.
expect(filters.some((f) => f.startsWith("nwr/tourism="))).toBe(true);
expect(filters.some((f) => f.startsWith("nwr/shop="))).toBe(true);
});
it("covers every selector value across the nine categories", () => {
const filters = osmiumTagFilters();
for (const cat of poiCategories) {
for (const sel of cat.selectors) {
const expr = filters.find((f) => f.startsWith(`nwr/${sel.key}=`));
expect(expr, `expected a filter for ${sel.key}`).toBeDefined();
expect(expr!.split("=")[1]!.split(",")).toContain(sel.value);
}
}
});
it("does not repeat a value for the same key", () => {
const filters = osmiumTagFilters();
for (const f of filters) {
const values = f.split("=")[1]!.split(",");
expect(new Set(values).size).toBe(values.length);
}
});
});
describe("profileOverlayDefaults", () => {
it("maps fastbike to cycling routes", () => {
expect(profileOverlayDefaults["fastbike"]).toContain("waymarked-cycling");

104
packages/map-core/src/poi.ts
View file

@ -1,9 +1,23 @@
/**
* A single OSM tag selector: an element matches the selector when its tags
* contain `key` mapped to `value`. Categories are unions of selectors.
*
* This structured form is the single source of truth for "what is a shelter":
* the POI extract pipeline renders it to an osmium tag-filter expression, and
* the flagship importer classifies each filtered element back to its
* category/categories. Nothing builds Overpass QL from these any more.
*/
export interface PoiSelector {
key: string;
value: string;
}
export interface PoiCategory {
id: string;
name: string;
icon: string;
color: string;
query: string;
selectors: PoiSelector[];
profiles?: string[];
}
@ -13,43 +27,66 @@ export const poiCategories: PoiCategory[] = [
name: "poi.drinkingWater",
icon: "\u{1F4A7}",
color: "#2563eb",
query: 'nwr["amenity"="drinking_water"];nwr["amenity"="water_point"];',
selectors: [
{ key: "amenity", value: "drinking_water" },
{ key: "amenity", value: "water_point" },
],
},
{
id: "shelter",
name: "poi.shelter",
icon: "\u{1F6D6}",
color: "#8B6D3A",
query: 'nwr["amenity"="shelter"];nwr["tourism"="wilderness_hut"];',
selectors: [
{ key: "amenity", value: "shelter" },
{ key: "tourism", value: "wilderness_hut" },
],
profiles: ["trekking"],
},
{
id: "camping",
name: "poi.camping",
icon: "\u26FA",
icon: "⛺",
color: "#059669",
query: 'nwr["tourism"="camp_site"];nwr["tourism"="caravan_site"];',
selectors: [
{ key: "tourism", value: "camp_site" },
{ key: "tourism", value: "caravan_site" },
],
},
{
id: "food",
name: "poi.food",
icon: "\u{1F37D}\uFE0F",
icon: "\u{1F37D}️",
color: "#dc2626",
query: 'nwr["amenity"="restaurant"];nwr["amenity"="cafe"];nwr["amenity"="fast_food"];nwr["amenity"="pub"];nwr["amenity"="biergarten"];',
selectors: [
{ key: "amenity", value: "restaurant" },
{ key: "amenity", value: "cafe" },
{ key: "amenity", value: "fast_food" },
{ key: "amenity", value: "pub" },
{ key: "amenity", value: "biergarten" },
],
},
{
id: "groceries",
name: "poi.groceries",
icon: "\u{1F6D2}",
color: "#f97316",
query: 'nwr["shop"="supermarket"];nwr["shop"="convenience"];nwr["shop"="bakery"];',
selectors: [
{ key: "shop", value: "supermarket" },
{ key: "shop", value: "convenience" },
{ key: "shop", value: "bakery" },
],
},
{
id: "bike_infra",
name: "poi.bikeInfra",
icon: "\u{1F527}",
color: "#8b5cf6",
query: 'nwr["amenity"="bicycle_parking"];nwr["amenity"="bicycle_repair_station"];nwr["amenity"="bicycle_rental"];',
selectors: [
{ key: "amenity", value: "bicycle_parking" },
{ key: "amenity", value: "bicycle_repair_station" },
{ key: "amenity", value: "bicycle_rental" },
],
profiles: ["fastbike", "safety"],
},
{
@ -57,14 +94,18 @@ export const poiCategories: PoiCategory[] = [
name: "poi.accommodation",
icon: "\u{1F3E8}",
color: "#0891b2",
query: 'nwr["tourism"="hotel"];nwr["tourism"="hostel"];nwr["tourism"="guest_house"];',
selectors: [
{ key: "tourism", value: "hotel" },
{ key: "tourism", value: "hostel" },
{ key: "tourism", value: "guest_house" },
],
},
{
id: "viewpoints",
name: "poi.viewpoints",
icon: "\u{1F441}\uFE0F",
icon: "\u{1F441}️",
color: "#9333ea",
query: 'nwr["tourism"="viewpoint"];',
selectors: [{ key: "tourism", value: "viewpoint" }],
profiles: ["trekking"],
},
{
@ -72,7 +113,7 @@ export const poiCategories: PoiCategory[] = [
name: "poi.toilets",
icon: "\u{1F6BB}",
color: "#6b7280",
query: 'nwr["amenity"="toilets"];',
selectors: [{ key: "amenity", value: "toilets" }],
},
];
@ -82,6 +123,43 @@ export function getCategoriesForProfile(profile: string): string[] {
.map((c) => c.id);
}
/**
* Return the ids of every category whose selectors match the given OSM tags.
* An element can belong to more than one category. Used by the POI import
* pipeline to classify each osmium-filtered element; the serving layer reads
* the stored category and never needs to re-classify.
*/
export function matchingCategoryIds(
tags: Record<string, string>,
categories: PoiCategory[] = poiCategories,
): string[] {
return categories
.filter((cat) => cat.selectors.some((s) => tags[s.key] === s.value))
.map((cat) => cat.id);
}
/**
* Render the categories' selectors as a deduplicated list of osmium
* `tags-filter` expressions, one per distinct key=value. Each expression
* matches nodes, ways and relations (`nwr/`), matching the previous Overpass
* `nwr[...]` selectors. Values for the same key are grouped so the filter is
* compact, e.g. `nwr/amenity=drinking_water,water_point`.
*/
export function osmiumTagFilters(
categories: PoiCategory[] = poiCategories,
): string[] {
// key -> ordered set of values (first-seen order, deduplicated)
const byKey = new Map<string, string[]>();
for (const cat of categories) {
for (const { key, value } of cat.selectors) {
const values = byKey.get(key) ?? [];
if (!values.includes(value)) values.push(value);
byKey.set(key, values);
}
}
return [...byKey.entries()].map(([key, values]) => `nwr/${key}=${values.join(",")}`);
}
/** Profile -> tile overlay mapping */
export const profileOverlayDefaults: Record<string, string[]> = {
fastbike: ["waymarked-cycling"],

8
packages/map-core/tsconfig.json
View file

@ -4,5 +4,11 @@
"outDir": "dist",
"rootDir": "src"
},
"include": ["src"]
"include": ["src"],
// *.sync.test.ts use node:fs to compare committed generated files
// (osmium-filters.txt, poi-selectors.sql) against the selectors. They run
// under vitest (node builtins provided) but are excluded from `tsc` so
// map-core keeps its zero-dependency, node-free type surface. All other
// tests are still typechecked.
"exclude": ["node_modules", "build", "dist", "src/**/*.sync.test.ts"]
}