poi-index: extract pipeline (BRouter host) + import job (flagship) + e2e

- poi-extract.sh: PBF download -> osmium tags-filter -> centroid NDJSON + manifest, published via the Caddy sidecar at vSwitch-only /poi/* - osmium-filters.txt + poi-selectors.sql generated from map-core selectors, guarded by sync tests so poiCategories stays the single source of truth - poi-import.sh: checksum verify -> classify into category rows -> 70% guard (--bootstrap override) -> atomic rename swap; node_exporter textfile metric for import outcome - systemd service+timer units for both halves (monthly, offset) - cd-infra.yml: ship infrastructure/scripts to the flagship - e2e: mock /api/pois instead of /api/overpass Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-12 22:48:32 +02:00 · 2026-07-12 22:48:32 +02:00 · e703843b9b
commit e703843b9b
parent b45e69885d
23 changed files with 747 additions and 31 deletions
--- a/infrastructure/scripts/README.md
+++ b/infrastructure/scripts/README.md
@ -0,0 +1,45 @@
+# Flagship operational scripts
+
+Deployed to `/opt/trails-cool/scripts/` by `cd-infra.yml`.
+
+## `backup-postgres.sh`
+Daily `pg_dump` + retention sweep. Run via cron (see the header).
+
+## POI index import (`poi-import.sh`)
+
+The "load where the database is" half of the [`poi-index`](../../openspec/changes/poi-index/)
+change. Pulls the artifact the BRouter host published, classifies each element
+into `planner.pois` category rows, and swaps it in atomically.
+
+- `poi-import.sh` — fetch (vSwitch) → checksum → load → classify → **guarded
+  atomic rename swap**. `--bootstrap` skips the 70% row-count guard for the
+  first import (when the live table is legitimately empty).
+- `poi-selectors.sql` — **generated** from `@trails-cool/map-core` selectors;
+  the classifier JOINs on it. Regenerate with
+  `npx tsx infrastructure/scripts/gen-poi-selectors-sql.ts` (CI sync test:
+  `packages/map-core/src/poi-selectors-sql.sync.test.ts`).
+- `poi-import.service` / `.timer` — monthly, offset a day after the extract.
+
+### First-time setup on the flagship
+
+```bash
+# 1. Create the empty live table from the Drizzle schema (idempotent).
+#    (Run from a checkout with DATABASE_URL pointed at production, or via the
+#    normal db:push deploy path.)
+pnpm db:push
+
+# 2. Install the timer.
+cp /opt/trails-cool/scripts/poi-import.{service,timer} /etc/systemd/system/
+systemctl daemon-reload
+systemctl enable --now poi-import.timer
+
+# 3. Bootstrap the first import (guard disabled — live table is empty).
+BROUTER_AUTH_TOKEN=$(grep BROUTER_AUTH_TOKEN /opt/trails-cool/.env | cut -d= -f2-) \
+  /opt/trails-cool/scripts/poi-import.sh --bootstrap
+```
+
+Prometheus picks up import outcome from node_exporter's textfile collector
+(`poi_import_last_status`, `…_last_success_timestamp_seconds`, `…_last_rows`)
+when `NODE_EXPORTER_TEXTFILE_DIR` is set (the unit sets it). Index size and age
+are additionally exposed by the planner at `/metrics`
+(`poi_index_rows{category}`, `poi_index_age_seconds`).
--- a/infrastructure/scripts/gen-poi-selectors-sql.ts
+++ b/infrastructure/scripts/gen-poi-selectors-sql.ts
@ -0,0 +1,36 @@
+/**
+ * Regenerate `poi-selectors.sql` from the POI category selectors in
+ * `@trails-cool/map-core` — the single source of truth. The flagship import
+ * script (`poi-import.sh`, bash + psql, no Node) loads this to classify each
+ * extracted OSM element into one row per matching category, mirroring
+ * `matchingCategoryIds`.
+ *
+ * Run after changing `poiCategories`:
+ *   npx tsx infrastructure/scripts/gen-poi-selectors-sql.ts
+ *
+ * `packages/map-core/src/poi-selectors-sql.sync.test.ts` fails CI on drift.
+ */
+import { writeFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { poiCategories } from "../../packages/map-core/src/poi.ts";
+
+function sqlLiteral(s: string): string {
+  return `'${s.replace(/'/g, "''")}'`;
+}
+
+const rows = poiCategories.flatMap((cat) =>
+  cat.selectors.map((s) => `  (${sqlLiteral(s.key)}, ${sqlLiteral(s.value)}, ${sqlLiteral(cat.id)})`),
+);
+
+const sql = `-- GENERATED from @trails-cool/map-core poiCategories. Do not edit by hand;
+-- regenerate with: npx tsx infrastructure/scripts/gen-poi-selectors-sql.ts
+-- Loaded by poi-import.sh inside the import transaction to classify each
+-- extracted OSM element into one row per matching category.
+CREATE TEMP TABLE poi_selector (key text, value text, category text) ON COMMIT DROP;
+INSERT INTO poi_selector (key, value, category) VALUES
+${rows.join(",\n")};
+`;
+
+const out = fileURLToPath(new URL("./poi-selectors.sql", import.meta.url));
+writeFileSync(out, sql);
+console.log(`Wrote ${out}`);
--- a/infrastructure/scripts/poi-import.service
+++ b/infrastructure/scripts/poi-import.service
@ -0,0 +1,19 @@
+# POI import — one-shot unit driven by poi-import.timer. Runs on the flagship
+# as root (the deploy user), pulling the artifact the BRouter host published and
+# swapping it into planner.pois. Install: copy to /etc/systemd/system/, then
+# `systemctl enable --now poi-import.timer`. See infrastructure/scripts/README
+# / the poi-index change docs. Requires BROUTER_AUTH_TOKEN in the environment
+# file below (reuse the deployed /opt/trails-cool/.env).
+[Unit]
+Description=Import the trails.cool POI index artifact into planner.pois
+Wants=network-online.target
+After=network-online.target docker.service
+
+[Service]
+Type=oneshot
+WorkingDirectory=/opt/trails-cool
+EnvironmentFile=/opt/trails-cool/.env
+# Optional: expose import-outcome metrics to node_exporter's textfile collector.
+Environment=NODE_EXPORTER_TEXTFILE_DIR=/var/lib/node_exporter/textfile_collector
+ExecStart=/opt/trails-cool/scripts/poi-import.sh
+TimeoutStartSec=2h
--- a/infrastructure/scripts/poi-import.sh
+++ b/infrastructure/scripts/poi-import.sh
@ -0,0 +1,162 @@
+#!/bin/bash
+# POI import — runs on the flagship. Pulls the POI artifact the BRouter host
+# published, classifies each element into category rows, and swaps it into the
+# live `planner.pois` table atomically. This is the "load where the database
+# is" half of the poi-index change.
+#
+# Serving is never interrupted: the new data is built in a side table and the
+# live table is replaced with a rename inside one transaction. A truncated
+# download can never blank the map — the swap is refused if the fresh dataset
+# has fewer than 70% of the live table's rows (override with --bootstrap on the
+# very first import, when the live table is legitimately empty).
+#
+# Prerequisites: the live table must already exist (created empty by
+# `pnpm db:push` from the Drizzle schema); curl, gunzip, sha256sum on the host.
+#
+# Usage:
+#   ./poi-import.sh [--bootstrap]
+#
+# Env:
+#   POI_ARTIFACT_BASE_URL   vSwitch URL of the published artifact dir.
+#                           Default: http://10.0.1.10:17777/poi
+#   BROUTER_AUTH_TOKEN      Shared secret Caddy requires (X-BRouter-Auth).
+#   COMPOSE_FILE            docker compose file. Default: /opt/trails-cool/docker-compose.yml
+#   NODE_EXPORTER_TEXTFILE_DIR  If set, an import-outcome .prom metric is written here.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SELECTORS_SQL="$SCRIPT_DIR/poi-selectors.sql"
+
+POI_ARTIFACT_BASE_URL="${POI_ARTIFACT_BASE_URL:-http://10.0.1.10:17777/poi}"
+COMPOSE_FILE="${COMPOSE_FILE:-/opt/trails-cool/docker-compose.yml}"
+BOOTSTRAP=0
+[ "${1:-}" = "--bootstrap" ] && BOOTSTRAP=1
+
+WORK_DIR="$(mktemp -d)"
+MANIFEST="$WORK_DIR/manifest.json"
+ARTIFACT="$WORK_DIR/pois.ndjson.gz"
+cleanup() { rm -rf "$WORK_DIR"; }
+trap cleanup EXIT
+
+log() { echo "[poi-import] $*"; }
+psql() { docker compose -f "$COMPOSE_FILE" exec -T postgres psql -U trails -d trails -v ON_ERROR_STOP=1 "$@"; }
+
+emit_metric() {
+  # Best-effort node_exporter textfile metric. status=1 success, 0 failure.
+  local status="$1" rows="${2:-0}"
+  [ -n "${NODE_EXPORTER_TEXTFILE_DIR:-}" ] || return 0
+  local now; now="$(date +%s)"
+  local tmp; tmp="$(mktemp)"
+  {
+    echo "# HELP poi_import_last_run_timestamp_seconds Unix time of the last POI import run."
+    echo "# TYPE poi_import_last_run_timestamp_seconds gauge"
+    echo "poi_import_last_run_timestamp_seconds $now"
+    echo "# HELP poi_import_last_status 1 if the last POI import swapped successfully, else 0."
+    echo "# TYPE poi_import_last_status gauge"
+    echo "poi_import_last_status $status"
+    echo "# HELP poi_import_last_rows Row count of the last successful POI import."
+    echo "# TYPE poi_import_last_rows gauge"
+    echo "poi_import_last_rows $rows"
+    if [ "$status" = "1" ]; then
+      echo "# HELP poi_import_last_success_timestamp_seconds Unix time of the last successful POI import."
+      echo "# TYPE poi_import_last_success_timestamp_seconds gauge"
+      echo "poi_import_last_success_timestamp_seconds $now"
+    fi
+  } > "$tmp"
+  mv "$tmp" "$NODE_EXPORTER_TEXTFILE_DIR/poi_import.prom"
+}
+
+fail() { log "ERROR: $*"; emit_metric 0; exit 1; }
+
+[ -s "$SELECTORS_SQL" ] || fail "$SELECTORS_SQL missing — regenerate with gen-poi-selectors-sql.ts"
+
+AUTH_HEADER=()
+[ -n "${BROUTER_AUTH_TOKEN:-}" ] && AUTH_HEADER=(-H "X-BRouter-Auth: ${BROUTER_AUTH_TOKEN}")
+
+# --- 1. Fetch manifest + artifact over the vSwitch ---------------------------
+log "fetching manifest from $POI_ARTIFACT_BASE_URL"
+curl --fail --silent --show-error "${AUTH_HEADER[@]}" -o "$MANIFEST" "$POI_ARTIFACT_BASE_URL/manifest.json" \
+  || fail "manifest fetch failed"
+EXPECTED_SHA="$(grep -o '"sha256"[[:space:]]*:[[:space:]]*"[0-9a-f]*"' "$MANIFEST" | grep -o '[0-9a-f]\{64\}')"
+MANIFEST_ROWS="$(grep -o '"row_count"[[:space:]]*:[[:space:]]*[0-9]*' "$MANIFEST" | grep -o '[0-9]*')"
+[ -n "$EXPECTED_SHA" ] || fail "manifest has no sha256"
+log "manifest: row_count=$MANIFEST_ROWS sha256=${EXPECTED_SHA:0:12}…"
+
+log "fetching artifact"
+curl --fail --silent --show-error "${AUTH_HEADER[@]}" -o "$ARTIFACT" "$POI_ARTIFACT_BASE_URL/pois.ndjson.gz" \
+  || fail "artifact fetch failed"
+
+# --- 2. Verify checksum ------------------------------------------------------
+ACTUAL_SHA="$(sha256sum "$ARTIFACT" | cut -d' ' -f1)"
+[ "$ACTUAL_SHA" = "$EXPECTED_SHA" ] || fail "checksum mismatch (got ${ACTUAL_SHA:0:12}…, want ${EXPECTED_SHA:0:12}…)"
+log "checksum verified"
+
+# --- 3. Load raw NDJSON into a staging import table --------------------------
+log "loading raw NDJSON"
+psql -c "CREATE TABLE IF NOT EXISTS planner.pois_raw_import (doc jsonb); TRUNCATE planner.pois_raw_import;"
+# Each gzip line is one JSON object. Load it as a single column value using
+# control-char delimiter/quote that never appear in JSON so the whole line
+# lands intact, then it's cast to jsonb.
+gunzip -c "$ARTIFACT" | psql -c \
+  "\copy planner.pois_raw_import (doc) FROM STDIN WITH (FORMAT csv, DELIMITER E'\x1f', QUOTE E'\x1e')" \
+  || fail "COPY failed"
+
+# --- 4. Build the staging table, classified into category rows ---------------
+# One transaction: temp selector table (from map-core) → pois_staging built
+# LIKE the live table so structure/defaults match the Drizzle schema exactly →
+# classified insert → indexes. The 70% guard + swap run in step 5.
+log "building classified staging table"
+psql <<SQL || fail "staging build failed"
+BEGIN;
+\\i $SELECTORS_SQL
+DROP TABLE IF EXISTS planner.pois_staging;
+CREATE TABLE planner.pois_staging (LIKE planner.pois INCLUDING DEFAULTS);
+INSERT INTO planner.pois_staging (osm_type, osm_id, category, name, geom, tags, imported_at)
+SELECT r.doc->>'osm_type',
+       r.doc->>'osm_id',
+       s.category,
+       r.doc->>'name',
+       ST_SetSRID(ST_MakePoint((r.doc->>'lon')::float8, (r.doc->>'lat')::float8), 4326),
+       COALESCE(r.doc->'tags', '{}'::jsonb),
+       now()
+FROM planner.pois_raw_import r
+JOIN poi_selector s ON r.doc->'tags'->>s.key = s.value;
+ALTER TABLE planner.pois_staging
+  ADD CONSTRAINT pois_staging_pkey PRIMARY KEY (osm_type, osm_id, category);
+CREATE INDEX pois_staging_geom_idx ON planner.pois_staging USING gist (geom);
+CREATE INDEX pois_staging_category_idx ON planner.pois_staging (category);
+COMMIT;
+SQL
+
+STAGING_ROWS="$(psql -tA -c "SELECT count(*) FROM planner.pois_staging;")"
+LIVE_ROWS="$(psql -tA -c "SELECT count(*) FROM planner.pois;")"
+log "staging=$STAGING_ROWS live=$LIVE_ROWS bootstrap=$BOOTSTRAP"
+[ "$STAGING_ROWS" -gt 0 ] || fail "staging is empty — refusing to swap"
+
+# --- 5. 70% guard + atomic swap ---------------------------------------------
+if [ "$BOOTSTRAP" -ne 1 ] && [ "$LIVE_ROWS" -gt 0 ]; then
+  # Integer 70% threshold: staging * 100 >= live * 70
+  if [ $(( STAGING_ROWS * 100 )) -lt $(( LIVE_ROWS * 70 )) ]; then
+    fail "guard tripped: staging ($STAGING_ROWS) < 70% of live ($LIVE_ROWS). No swap. Use --bootstrap to override."
+  fi
+fi
+
+log "swapping staging into live (atomic)"
+psql <<'SQL' || fail "swap failed"
+BEGIN;
+DROP TABLE IF EXISTS planner.pois_old;
+ALTER TABLE planner.pois RENAME TO pois_old;
+ALTER TABLE planner.pois_staging RENAME TO pois;
+-- Rename constraints/indexes to the canonical Drizzle names so a later
+-- `db:push` sees no drift. pois_old still holds the old canonical names until
+-- it's dropped, so free them first.
+DROP TABLE planner.pois_old;
+ALTER TABLE planner.pois RENAME CONSTRAINT pois_staging_pkey TO pois_osm_type_osm_id_category_pk;
+ALTER INDEX planner.pois_staging_geom_idx RENAME TO pois_geom_idx;
+ALTER INDEX planner.pois_staging_category_idx RENAME TO pois_category_idx;
+COMMIT;
+SQL
+
+psql -c "DROP TABLE IF EXISTS planner.pois_raw_import;"
+log "done: live table now has $STAGING_ROWS rows"
+emit_metric 1 "$STAGING_ROWS"
--- a/infrastructure/scripts/poi-import.timer
+++ b/infrastructure/scripts/poi-import.timer
@ -0,0 +1,12 @@
+# Monthly POI import, offset a day after the BRouter host's extract (which runs
+# on the 1st) so the fresh artifact is already published when this pulls it.
+[Unit]
+Description=Monthly trails.cool POI index import
+
+[Timer]
+OnCalendar=*-*-02 04:00:00
+Persistent=true
+RandomizedDelaySec=30m
+
+[Install]
+WantedBy=timers.target
--- a/infrastructure/scripts/poi-selectors.sql
+++ b/infrastructure/scripts/poi-selectors.sql
@ -0,0 +1,28 @@
+-- GENERATED from @trails-cool/map-core poiCategories. Do not edit by hand;
+-- regenerate with: npx tsx infrastructure/scripts/gen-poi-selectors-sql.ts
+-- Loaded by poi-import.sh inside the import transaction to classify each
+-- extracted OSM element into one row per matching category.
+CREATE TEMP TABLE poi_selector (key text, value text, category text) ON COMMIT DROP;
+INSERT INTO poi_selector (key, value, category) VALUES
+  ('amenity', 'drinking_water', 'drinking_water'),
+  ('amenity', 'water_point', 'drinking_water'),
+  ('amenity', 'shelter', 'shelter'),
+  ('tourism', 'wilderness_hut', 'shelter'),
+  ('tourism', 'camp_site', 'camping'),
+  ('tourism', 'caravan_site', 'camping'),
+  ('amenity', 'restaurant', 'food'),
+  ('amenity', 'cafe', 'food'),
+  ('amenity', 'fast_food', 'food'),
+  ('amenity', 'pub', 'food'),
+  ('amenity', 'biergarten', 'food'),
+  ('shop', 'supermarket', 'groceries'),
+  ('shop', 'convenience', 'groceries'),
+  ('shop', 'bakery', 'groceries'),
+  ('amenity', 'bicycle_parking', 'bike_infra'),
+  ('amenity', 'bicycle_repair_station', 'bike_infra'),
+  ('amenity', 'bicycle_rental', 'bike_infra'),
+  ('tourism', 'hotel', 'accommodation'),
+  ('tourism', 'hostel', 'accommodation'),
+  ('tourism', 'guest_house', 'accommodation'),
+  ('tourism', 'viewpoint', 'viewpoints'),
+  ('amenity', 'toilets', 'toilets');