Public Overpass instances go bad without warning — we just lived through a day where overpass.private.coffee (our single upstream) returned 504s after 60s while lz4.overpass-api.de served the same query in 0.6s. Survive that by trying a list of upstreams in order until one responds usefully. Server changes: - OVERPASS_URLS env — comma-separated list, tried in order. Falls back to OVERPASS_URL for backward compat, then to a built-in default pair (lz4.overpass-api.de, overpass-api.de). - Per-upstream timeout of 10s via AbortSignal.timeout, so a saturated instance fails over in seconds, not minutes. - Client abort propagation via AbortSignal.any — if the browser cancels (e.g. user panned the map), the in-flight upstream fetch is aborted too. Stops wasting upstream capacity on requests nobody wants. - Rate-limited-body detection (Overpass returns 200 with a `rate_limited` marker when throttling) triggers failover. - Per-upstream labels on overpass_upstream_requests_total and overpass_upstream_duration_seconds so the dashboard can break out health + latency by instance. Histogram buckets extended to 30s. Compose: OVERPASS_URLS passthrough with the default pair hard-wired, overridable via SOPS. Tests: 8 cases covering the new fetchWithFailover helper — happy path, 5xx failover, rate_limited failover, network error failover, timeout tagging, all-upstreams-fail, client abort stops the loop, per-attempt latency observation. Follow-ups left out of scope: - Client-side debounce (UI concern). - Moving `.observe()` after body read for accuracy in measurement. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
209 lines
6.5 KiB
TypeScript
209 lines
6.5 KiB
TypeScript
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
|
|
|
// Metrics module pulls in prom-client and registers processes-wide
|
|
// metrics on import; mock it out so tests aren't entangled with
|
|
// the registry and can assert on label values instead.
|
|
vi.mock("~/lib/metrics.server", () => ({
|
|
overpassCacheEvents: { inc: vi.fn() },
|
|
overpassCacheSize: { set: vi.fn() },
|
|
overpassUpstreamDuration: { observe: vi.fn() },
|
|
overpassUpstreamRequests: { inc: vi.fn() },
|
|
}));
|
|
|
|
import { fetchWithFailover } from "./api.overpass";
|
|
import {
|
|
overpassUpstreamRequests,
|
|
overpassUpstreamDuration,
|
|
} from "~/lib/metrics.server";
|
|
|
|
const URL_A = "https://a.example/api/interpreter";
|
|
const URL_B = "https://b.example/api/interpreter";
|
|
const URL_C = "https://c.example/api/interpreter";
|
|
|
|
function makeResponse(
|
|
body: string,
|
|
init: { status?: number; headers?: Record<string, string> } = {},
|
|
): Response {
|
|
return new Response(body, {
|
|
status: init.status ?? 200,
|
|
headers: init.headers ?? { "content-type": "application/json" },
|
|
});
|
|
}
|
|
|
|
beforeEach(() => {
|
|
// `clearAllMocks` zeroes call-history on module-level mocks like
|
|
// `overpassUpstreamRequests.inc`, which would otherwise accumulate
|
|
// across test cases and break assertions that inspect call counts.
|
|
vi.clearAllMocks();
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.unstubAllGlobals();
|
|
});
|
|
|
|
describe("fetchWithFailover", () => {
|
|
it("returns the first upstream's response when it succeeds", async () => {
|
|
const fetchSpy = vi.fn().mockResolvedValueOnce(makeResponse('{"ok":1}'));
|
|
vi.stubGlobal("fetch", fetchSpy);
|
|
|
|
const result = await fetchWithFailover(
|
|
"data=...",
|
|
new AbortController().signal,
|
|
[URL_A, URL_B],
|
|
);
|
|
|
|
expect(result?.status).toBe(200);
|
|
expect(result?.body).toBe('{"ok":1}');
|
|
expect(fetchSpy).toHaveBeenCalledTimes(1);
|
|
expect(fetchSpy).toHaveBeenCalledWith(URL_A, expect.any(Object));
|
|
expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({
|
|
upstream: "a.example",
|
|
status: "200",
|
|
});
|
|
});
|
|
|
|
it("falls over to the next upstream on non-2xx", async () => {
|
|
const fetchSpy = vi
|
|
.fn()
|
|
.mockResolvedValueOnce(makeResponse("gateway timeout", { status: 504 }))
|
|
.mockResolvedValueOnce(makeResponse('{"elements":[]}'));
|
|
vi.stubGlobal("fetch", fetchSpy);
|
|
|
|
const result = await fetchWithFailover(
|
|
"data=...",
|
|
new AbortController().signal,
|
|
[URL_A, URL_B],
|
|
);
|
|
|
|
expect(result?.body).toBe('{"elements":[]}');
|
|
expect(fetchSpy).toHaveBeenCalledTimes(2);
|
|
expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({
|
|
upstream: "a.example",
|
|
status: "504",
|
|
});
|
|
expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({
|
|
upstream: "b.example",
|
|
status: "200",
|
|
});
|
|
});
|
|
|
|
it("falls over when the body carries Overpass's rate_limited runtime error", async () => {
|
|
// Overpass returns HTTP 200 with a `rate_limited` marker in the
|
|
// body when it's throttling. Treat that as an upstream failure.
|
|
const fetchSpy = vi
|
|
.fn()
|
|
.mockResolvedValueOnce(makeResponse('{"error":"rate_limited"}'))
|
|
.mockResolvedValueOnce(makeResponse('{"elements":[]}'));
|
|
vi.stubGlobal("fetch", fetchSpy);
|
|
|
|
const result = await fetchWithFailover(
|
|
"data=...",
|
|
new AbortController().signal,
|
|
[URL_A, URL_B],
|
|
);
|
|
|
|
expect(result?.body).toBe('{"elements":[]}');
|
|
expect(fetchSpy).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it("falls over on network / timeout errors", async () => {
|
|
const fetchSpy = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(new Error("connect ECONNREFUSED"))
|
|
.mockResolvedValueOnce(makeResponse('{"elements":[]}'));
|
|
vi.stubGlobal("fetch", fetchSpy);
|
|
|
|
const result = await fetchWithFailover(
|
|
"data=...",
|
|
new AbortController().signal,
|
|
[URL_A, URL_B],
|
|
);
|
|
|
|
expect(result?.body).toBe('{"elements":[]}');
|
|
expect(fetchSpy).toHaveBeenCalledTimes(2);
|
|
expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({
|
|
upstream: "a.example",
|
|
status: "error",
|
|
});
|
|
});
|
|
|
|
it("tags TimeoutError distinctly from other errors", async () => {
|
|
const timeoutErr = new Error("aborted");
|
|
timeoutErr.name = "TimeoutError";
|
|
const fetchSpy = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(timeoutErr)
|
|
.mockResolvedValueOnce(makeResponse('{"elements":[]}'));
|
|
vi.stubGlobal("fetch", fetchSpy);
|
|
|
|
await fetchWithFailover(
|
|
"data=...",
|
|
new AbortController().signal,
|
|
[URL_A, URL_B],
|
|
);
|
|
|
|
expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({
|
|
upstream: "a.example",
|
|
status: "timeout",
|
|
});
|
|
});
|
|
|
|
it("returns null when every upstream fails", async () => {
|
|
const fetchSpy = vi
|
|
.fn()
|
|
.mockResolvedValueOnce(makeResponse("bad", { status: 500 }))
|
|
.mockResolvedValueOnce(makeResponse("bad", { status: 502 }))
|
|
.mockResolvedValueOnce(makeResponse("bad", { status: 503 }));
|
|
vi.stubGlobal("fetch", fetchSpy);
|
|
|
|
const result = await fetchWithFailover(
|
|
"data=...",
|
|
new AbortController().signal,
|
|
[URL_A, URL_B, URL_C],
|
|
);
|
|
|
|
expect(result).toBeNull();
|
|
expect(fetchSpy).toHaveBeenCalledTimes(3);
|
|
});
|
|
|
|
it("stops iterating when the client aborts, rethrows, and doesn't try later upstreams", async () => {
|
|
const controller = new AbortController();
|
|
const abortErr = new DOMException("aborted", "AbortError");
|
|
const fetchSpy = vi.fn().mockImplementationOnce(async () => {
|
|
controller.abort();
|
|
throw abortErr;
|
|
});
|
|
vi.stubGlobal("fetch", fetchSpy);
|
|
|
|
await expect(
|
|
fetchWithFailover("data=...", controller.signal, [URL_A, URL_B]),
|
|
).rejects.toBe(abortErr);
|
|
|
|
// Only A was attempted; B never got a chance.
|
|
expect(fetchSpy).toHaveBeenCalledTimes(1);
|
|
expect(overpassUpstreamRequests.inc).toHaveBeenCalledWith({
|
|
upstream: "a.example",
|
|
status: "client-abort",
|
|
});
|
|
});
|
|
|
|
it("records latency with the upstream label on every attempt (success and failure)", async () => {
|
|
const fetchSpy = vi
|
|
.fn()
|
|
.mockResolvedValueOnce(makeResponse("bad", { status: 503 }))
|
|
.mockResolvedValueOnce(makeResponse('{"elements":[]}'));
|
|
vi.stubGlobal("fetch", fetchSpy);
|
|
|
|
await fetchWithFailover(
|
|
"data=...",
|
|
new AbortController().signal,
|
|
[URL_A, URL_B],
|
|
);
|
|
|
|
const calls = vi.mocked(overpassUpstreamDuration.observe).mock.calls;
|
|
const upstreams = calls.map(
|
|
(c) => (c[0] as unknown as { upstream: string }).upstream,
|
|
);
|
|
expect(upstreams).toEqual(["a.example", "b.example"]);
|
|
});
|
|
});
|