poi-index: extract pipeline (BRouter host) + import job (flagship) + e2e

- poi-extract.sh: PBF download -> osmium tags-filter -> centroid NDJSON +
  manifest, published via the Caddy sidecar at vSwitch-only /poi/*
- osmium-filters.txt + poi-selectors.sql generated from map-core selectors,
  guarded by sync tests so poiCategories stays the single source of truth
- poi-import.sh: checksum verify -> classify into category rows ->
  70% guard (--bootstrap override) -> atomic rename swap; node_exporter
  textfile metric for import outcome
- systemd service+timer units for both halves (monthly, offset)
- cd-infra.yml: ship infrastructure/scripts to the flagship
- e2e: mock /api/pois instead of /api/overpass

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ullrich Schäfer 2026-07-12 22:48:32 +02:00
parent b45e69885d
commit e703843b9b
No known key found for this signature in database
GPG key ID: A32FF691A0F752D9
23 changed files with 747 additions and 31 deletions

View file

@ -0,0 +1,22 @@
import { describe, it, expect } from "vitest";
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { osmiumTagFilters } from "./poi.ts";
// The BRouter host reads the committed `osmium-filters.txt` at extract time —
// it has no Node runtime to derive filters from map-core. This test fails CI
// if the file drifts from the selectors, so `poiCategories` stays the single
// source of truth. Regenerate with:
// npx tsx infrastructure/brouter-host/poi-extract/gen-osmium-filters.ts
describe("osmium-filters.txt", () => {
it("matches the filters derived from poiCategories", () => {
const path = fileURLToPath(
new URL(
"../../../infrastructure/brouter-host/poi-extract/osmium-filters.txt",
import.meta.url,
),
);
const committed = readFileSync(path, "utf8").trimEnd();
expect(committed).toBe(osmiumTagFilters().join("\n"));
});
});

View file

@ -0,0 +1,27 @@
import { describe, it, expect } from "vitest";
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { poiCategories } from "./poi.ts";
// The flagship import script (poi-import.sh, bash + psql, no Node) loads the
// committed poi-selectors.sql to classify extracted elements into category
// rows. This test fails CI if that file drifts from the map-core selectors, so
// poiCategories stays the single source of truth. Regenerate with:
// npx tsx infrastructure/scripts/gen-poi-selectors-sql.ts
describe("poi-selectors.sql", () => {
it("contains a VALUES row for every selector/category pair", () => {
const path = fileURLToPath(
new URL("../../../infrastructure/scripts/poi-selectors.sql", import.meta.url),
);
const sql = readFileSync(path, "utf8");
const expected = poiCategories.flatMap((cat) =>
cat.selectors.map((s) => `('${s.key}', '${s.value}', '${cat.id}')`),
);
for (const row of expected) {
expect(sql).toContain(row);
}
// No extra/stale rows: the count of VALUES tuples equals the selector count.
const tupleCount = (sql.match(/\(\s*'[^']+',\s*'[^']+',\s*'[^']+'\s*\)/g) ?? []).length;
expect(tupleCount).toBe(expected.length);
});
});