trails/apps/journal/app/lib/metrics.server.ts
Ullrich Schäfer cd73093beb fix(journal): bound Prometheus route label cardinality
The httpRequestDuration histogram labeled every request with the raw URL
path (`url.split("?")[0]`), so every distinct path — `/activities/<uuid>`,
`/routes/<uuid>`, probed usernames, crawler junk — became a permanent
label-set. prom-client never evicts label-sets, so RSS grew linearly for
the life of the process and reset only on restart. On the flagship this
showed as ~25 MiB/day of unbounded journal memory growth; a live
`/api/metrics` scrape held 2,655 histogram series across 291 distinct
`route` values (344 KB body), dominated by per-UUID activity paths.

Add `normalizeRoute()` in metrics.server.ts: it collapses dynamic path
segments back to the `:param` templates declared in app/routes.ts (UUID
and numeric segments -> `:id`; `:username`/`:provider` slots keyed by
their preceding static segment), strips query/fragment, and enforces a
hard cap (MAX_ROUTES) that funnels anything beyond the known route table
into a single `/other` bucket as an absolute backstop. Cardinality is now
proportional to the route table, not to traffic.

Logs still record the raw path (`logger.info`) — only the metric label is
bounded.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-18 15:58:01 +02:00

113 lines
4.2 KiB
TypeScript

import client from "prom-client";
// React Router's build sometimes produces two module instances of this
// file (server entry + route module graph). prom-client's registry is a
// process-wide singleton, so re-running the `new Gauge(...)` / `new
// Histogram(...)` / `collectDefaultMetrics()` calls on the second load
// throws "metric already registered". Guard everything via
// `getSingleMetric` so a second module load reuses the existing objects.
function getOrCreate<T extends client.Metric<string>>(
name: string,
create: () => T,
): T {
const existing = client.register.getSingleMetric(name);
if (existing) return existing as T;
return create();
}
// Default process/Node metrics: registered exactly once.
if (!client.register.getSingleMetric("process_cpu_user_seconds_total")) {
client.collectDefaultMetrics();
}
export const httpRequestDuration = getOrCreate("http_request_duration_seconds", () =>
new client.Histogram({
name: "http_request_duration_seconds",
help: "Duration of HTTP requests in seconds",
labelNames: ["method", "route", "status"] as const,
buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
}),
);
export const demoBotSyntheticRoutesTotal = getOrCreate(
"demo_bot_synthetic_routes_total",
() =>
new client.Gauge({
name: "demo_bot_synthetic_routes_total",
help: "Total synthetic demo-bot routes currently stored",
}),
);
export const demoBotSyntheticActivitiesTotal = getOrCreate(
"demo_bot_synthetic_activities_total",
() =>
new client.Gauge({
name: "demo_bot_synthetic_activities_total",
help: "Total synthetic demo-bot activities currently stored",
}),
);
export const registry = client.register;
// --- Route label normalization -------------------------------------------
//
// The `route` label on httpRequestDuration MUST be bounded. prom-client
// stores one label-set per distinct combination and NEVER evicts, so
// feeding it raw request paths (`/activities/<uuid>`, probed usernames,
// crawler junk) leaks memory linearly for the life of the process — it
// only resets on restart. `normalizeRoute` collapses dynamic path
// segments back to the `:param` templates declared in app/routes.ts so
// cardinality stays proportional to the route table, not to traffic.
const UUID_RE =
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
// A dynamic segment is keyed by the static segment that precedes it,
// mirroring the non-id `:param` slots in app/routes.ts. (`:id` params are
// all UUIDs and get caught by UUID_RE on their own.)
const DYNAMIC_AFTER = new Map<string, string>([
["users", ":username"], // /users/:username, /api/users/:username/...
["connect", ":provider"], // /api/sync/connect/:provider
["callback", ":provider"], // /api/sync/callback/:provider
["disconnect", ":provider"], // /api/sync/disconnect/:provider
["webhook", ":provider"], // /api/sync/webhook/:provider
["push", ":provider"], // /api/sync/push/:provider/:routeId
]);
// Absolute backstop. Real route templates number well under this; once we
// have seen this many distinct normalized routes (e.g. an aggressive
// scanner hitting paths that match no `:param` rule), everything new
// collapses to a single `/other` bucket so cardinality can never run away.
const MAX_ROUTES = 200;
const seenRoutes = new Set<string>();
/** Collapse a request path's dynamic segments to a bounded route template. */
export function normalizeRoute(pathname: string): string {
const path = (pathname.split("?")[0] ?? "/").split("#")[0] || "/";
if (path === "/") return "/";
const out: string[] = [];
for (const seg of path.split("/")) {
if (seg === "") continue;
const prev = out.length > 0 ? out[out.length - 1]! : "";
if (UUID_RE.test(seg) || /^\d+$/.test(seg)) {
out.push(":id");
} else if (DYNAMIC_AFTER.has(prev)) {
out.push(DYNAMIC_AFTER.get(prev)!);
} else {
out.push(seg);
}
}
const normalized = "/" + out.join("/");
if (seenRoutes.has(normalized)) return normalized;
if (seenRoutes.size >= MAX_ROUTES) return "/other";
seenRoutes.add(normalized);
return normalized;
}
/** Test-only: clear the distinct-route tracking set. */
export function _resetRouteNormalizationForTest(): void {
seenRoutes.clear();
}