poi-index: extract pipeline (BRouter host) + import job (flagship) + e2e
- poi-extract.sh: PBF download -> osmium tags-filter -> centroid NDJSON + manifest, published via the Caddy sidecar at vSwitch-only /poi/* - osmium-filters.txt + poi-selectors.sql generated from map-core selectors, guarded by sync tests so poiCategories stays the single source of truth - poi-import.sh: checksum verify -> classify into category rows -> 70% guard (--bootstrap override) -> atomic rename swap; node_exporter textfile metric for import outcome - systemd service+timer units for both halves (monthly, offset) - cd-infra.yml: ship infrastructure/scripts to the flagship - e2e: mock /api/pois instead of /api/overpass Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
b45e69885d
commit
e703843b9b
23 changed files with 747 additions and 31 deletions
22
packages/map-core/src/osmium-filters.sync.test.ts
Normal file
22
packages/map-core/src/osmium-filters.sync.test.ts
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
import { describe, it, expect } from "vitest";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { osmiumTagFilters } from "./poi.ts";
|
||||
|
||||
// The BRouter host reads the committed `osmium-filters.txt` at extract time —
|
||||
// it has no Node runtime to derive filters from map-core. This test fails CI
|
||||
// if the file drifts from the selectors, so `poiCategories` stays the single
|
||||
// source of truth. Regenerate with:
|
||||
// npx tsx infrastructure/brouter-host/poi-extract/gen-osmium-filters.ts
|
||||
describe("osmium-filters.txt", () => {
|
||||
it("matches the filters derived from poiCategories", () => {
|
||||
const path = fileURLToPath(
|
||||
new URL(
|
||||
"../../../infrastructure/brouter-host/poi-extract/osmium-filters.txt",
|
||||
import.meta.url,
|
||||
),
|
||||
);
|
||||
const committed = readFileSync(path, "utf8").trimEnd();
|
||||
expect(committed).toBe(osmiumTagFilters().join("\n"));
|
||||
});
|
||||
});
|
||||
27
packages/map-core/src/poi-selectors-sql.sync.test.ts
Normal file
27
packages/map-core/src/poi-selectors-sql.sync.test.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
import { describe, it, expect } from "vitest";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { poiCategories } from "./poi.ts";
|
||||
|
||||
// The flagship import script (poi-import.sh, bash + psql, no Node) loads the
|
||||
// committed poi-selectors.sql to classify extracted elements into category
|
||||
// rows. This test fails CI if that file drifts from the map-core selectors, so
|
||||
// poiCategories stays the single source of truth. Regenerate with:
|
||||
// npx tsx infrastructure/scripts/gen-poi-selectors-sql.ts
|
||||
describe("poi-selectors.sql", () => {
|
||||
it("contains a VALUES row for every selector/category pair", () => {
|
||||
const path = fileURLToPath(
|
||||
new URL("../../../infrastructure/scripts/poi-selectors.sql", import.meta.url),
|
||||
);
|
||||
const sql = readFileSync(path, "utf8");
|
||||
const expected = poiCategories.flatMap((cat) =>
|
||||
cat.selectors.map((s) => `('${s.key}', '${s.value}', '${cat.id}')`),
|
||||
);
|
||||
for (const row of expected) {
|
||||
expect(sql).toContain(row);
|
||||
}
|
||||
// No extra/stale rows: the count of VALUES tuples equals the selector count.
|
||||
const tupleCount = (sql.match(/\(\s*'[^']+',\s*'[^']+',\s*'[^']+'\s*\)/g) ?? []).length;
|
||||
expect(tupleCount).toBe(expected.length);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue