diff --git a/apps/journal/app/hooks/useSurfaceBackfill.ts b/apps/journal/app/hooks/useSurfaceBackfill.ts new file mode 100644 index 0000000..ea0aacc --- /dev/null +++ b/apps/journal/app/hooks/useSurfaceBackfill.ts @@ -0,0 +1,38 @@ +import { useEffect } from "react"; +import { useRevalidator } from "react-router"; + +/** + * Live-updates a route/activity detail page when its async surface backfill + * completes. Subscribes to `/api/events` and, on a `surface_breakdown` event + * matching this row, re-runs the loader (so the bars appear without a reload). + * + * Enable only while it's worth it — i.e. the viewer is the owner (the backfill + * emits to the owner's user stream) and the breakdown isn't present yet. Once + * the breakdown lands, the loader revalidates, `enabled` flips false, and the + * connection is torn down. + */ +export function useSurfaceBackfillUpdates( + kind: "route" | "activity", + id: string, + enabled: boolean, +): void { + const revalidator = useRevalidator(); + useEffect(() => { + if (!enabled) return; + if (typeof EventSource === "undefined") return; + const es = new EventSource("/api/events"); + const onEvent = (e: MessageEvent) => { + try { + const parsed = JSON.parse(e.data) as { kind?: string; id?: string }; + if (parsed.kind === kind && parsed.id === id) revalidator.revalidate(); + } catch { + // Malformed payload — ignore. + } + }; + es.addEventListener("surface_breakdown", onEvent as EventListener); + return () => { + es.removeEventListener("surface_breakdown", onEvent as EventListener); + es.close(); + }; + }, [kind, id, enabled, revalidator]); +} diff --git a/apps/journal/app/jobs/payloads.ts b/apps/journal/app/jobs/payloads.ts index 6178f5c..3fa63f6 100644 --- a/apps/journal/app/jobs/payloads.ts +++ b/apps/journal/app/jobs/payloads.ts @@ -26,6 +26,7 @@ export interface JobPayloads { "poll-remote-actor": { actorIri: string }; "poll-remote-outboxes": void; "send-welcome-email": { email: string; username: string }; + "surface-backfill": { kind: "route" | "activity"; id: string }; } export type JobName = keyof JobPayloads; diff --git a/apps/journal/app/jobs/surface-backfill.test.ts b/apps/journal/app/jobs/surface-backfill.test.ts new file mode 100644 index 0000000..41caec3 --- /dev/null +++ b/apps/journal/app/jobs/surface-backfill.test.ts @@ -0,0 +1,56 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +const execute = vi.fn(); +vi.mock("../lib/db.ts", () => ({ getDb: () => ({ execute }) })); + +const fetchWaysInBbox = vi.fn(); +vi.mock("../lib/overpass-ways.server.ts", () => ({ fetchWaysInBbox: (...a: unknown[]) => fetchWaysInBbox(...a) })); + +const emitTo = vi.fn(); +vi.mock("../lib/events.server.ts", () => ({ emitTo: (...a: unknown[]) => emitTo(...a) })); + +vi.mock("../lib/logger.server.ts", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } })); + +import { runSurfaceBackfill } from "./surface-backfill.ts"; + +const lineString = (coords: number[][]) => JSON.stringify({ type: "LineString", coordinates: coords }); + +beforeEach(() => { + execute.mockReset(); + fetchWaysInBbox.mockReset(); + emitTo.mockReset(); +}); + +describe("runSurfaceBackfill", () => { + it("matches ways, stores the breakdown, and emits to the owner", async () => { + execute.mockResolvedValueOnce([ + { geojson: lineString([[0, 0], [0.001, 0], [0.002, 0]]), ownerId: "user-1", hasBreakdown: false }, + ]); // SELECT + execute.mockResolvedValueOnce(undefined); // UPDATE + fetchWaysInBbox.mockResolvedValue([ + { highway: "residential", surface: "asphalt", geometry: [{ lat: 0, lon: 0 }, { lat: 0, lon: 0.01 }] }, + ]); + + await runSurfaceBackfill("activity", "act-1"); + + expect(fetchWaysInBbox).toHaveBeenCalledOnce(); + expect(execute).toHaveBeenCalledTimes(2); // SELECT + UPDATE + expect(emitTo).toHaveBeenCalledWith("user-1", "surface_breakdown", { kind: "activity", id: "act-1" }); + }); + + it("skips a row that already has a breakdown", async () => { + execute.mockResolvedValueOnce([{ geojson: lineString([[0, 0], [1, 0]]), ownerId: "u", hasBreakdown: true }]); + await runSurfaceBackfill("route", "r-1"); + expect(fetchWaysInBbox).not.toHaveBeenCalled(); + expect(execute).toHaveBeenCalledTimes(1); // only the SELECT + expect(emitTo).not.toHaveBeenCalled(); + }); + + it("does not store or emit when Overpass returns no ways", async () => { + execute.mockResolvedValueOnce([{ geojson: lineString([[0, 0], [0.001, 0]]), ownerId: "u", hasBreakdown: false }]); + fetchWaysInBbox.mockResolvedValue([]); + await runSurfaceBackfill("activity", "a-2"); + expect(execute).toHaveBeenCalledTimes(1); // SELECT only + expect(emitTo).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/journal/app/jobs/surface-backfill.ts b/apps/journal/app/jobs/surface-backfill.ts new file mode 100644 index 0000000..4087045 --- /dev/null +++ b/apps/journal/app/jobs/surface-backfill.ts @@ -0,0 +1,101 @@ +import { sql } from "drizzle-orm"; +import { defineJournalJob } from "./payloads.ts"; +import { getDb } from "../lib/db.ts"; +import { computeSurfaceBreakdown } from "@trails-cool/map-core"; +import { fetchWaysInBbox, type Bbox } from "../lib/overpass-ways.server.ts"; +import { matchSurfaces } from "../lib/surface-match.server.ts"; +import { emitTo } from "../lib/events.server.ts"; +import { logger } from "../lib/logger.server.ts"; + +// Cap coordinates fed to the matcher; the breakdown still weights by the actual +// segment length, so downsampling barely shifts proportions. +const MAX_COORDS = 250; +const BBOX_PAD_DEG = 0.0008; // ~80 m, so ways just off the line are considered + +/** + * Derive a surface/waytype breakdown for a route/activity that has geometry but + * no breakdown yet (imports, uploads, pre-existing rows), by map-matching its + * geometry to OSM ways via Overpass. Idempotent (skips if already set), + * best-effort (Overpass failure throws → pg-boss retries; an empty/oversized + * bbox is a no-op). On success it stores the breakdown and pushes an SSE event + * to the owner so an open detail page fills the bars in live. + */ +export const surfaceBackfillJob = defineJournalJob({ + name: "surface-backfill", + retryLimit: 3, + expireInSeconds: 120, + async handler(job) { + const batch = Array.isArray(job) ? job : [job]; + for (const item of batch) { + await runSurfaceBackfill(item.data.kind, item.data.id); + } + }, +}); + +export async function runSurfaceBackfill(kind: "route" | "activity", id: string): Promise { + const db = getDb(); + const tableName = kind === "route" ? sql`journal.routes` : sql`journal.activities`; + + const loaded = (await db.execute(sql` + SELECT ST_AsGeoJSON(geom) AS geojson, + owner_id AS "ownerId", + (surface_breakdown IS NOT NULL) AS "hasBreakdown" + FROM ${tableName} WHERE id = ${id} LIMIT 1 + `)) as unknown as Array<{ geojson: string | null; ownerId: string | null; hasBreakdown: boolean }>; + const row = loaded[0]; + + if (!row) { + logger.warn({ kind, id }, "surface-backfill: row not found"); + return; + } + if (row.hasBreakdown) return; // idempotent + if (!row.geojson) return; // no geometry to match + + let coords: number[][]; + try { + coords = (JSON.parse(row.geojson) as { coordinates?: number[][] }).coordinates ?? []; + } catch { + return; + } + if (coords.length < 2) return; + + if (coords.length > MAX_COORDS) { + const stride = Math.ceil(coords.length / MAX_COORDS); + const out: number[][] = []; + for (let i = 0; i < coords.length; i += stride) out.push(coords[i]!); + const last = coords[coords.length - 1]!; + if (out[out.length - 1] !== last) out.push(last); + coords = out; + } + + let south = coords[0]![1]!, north = south, west = coords[0]![0]!, east = west; + for (const c of coords) { + const lon = c[0]!, lat = c[1]!; + if (lat < south) south = lat; + if (lat > north) north = lat; + if (lon < west) west = lon; + if (lon > east) east = lon; + } + const bbox: Bbox = { + south: south - BBOX_PAD_DEG, + west: west - BBOX_PAD_DEG, + north: north + BBOX_PAD_DEG, + east: east + BBOX_PAD_DEG, + }; + + const ways = await fetchWaysInBbox(bbox); + if (ways.length === 0) { + logger.info({ kind, id }, "surface-backfill: no ways (bbox empty/oversized) — skipping"); + return; + } + + const { surfaces, highways } = matchSurfaces(coords, ways); + const breakdown = computeSurfaceBreakdown(coords, surfaces, highways); + if (Object.keys(breakdown.surface).length === 0 && Object.keys(breakdown.highway).length === 0) return; + + await db.execute(sql` + UPDATE ${tableName} SET surface_breakdown = ${JSON.stringify(breakdown)}::jsonb WHERE id = ${id} + `); + if (row.ownerId) emitTo(row.ownerId, "surface_breakdown", { kind, id }); + logger.info({ kind, id }, "surface-backfill: stored breakdown"); +} diff --git a/apps/journal/app/lib/activities.server.ts b/apps/journal/app/lib/activities.server.ts index c1e3315..6a97b80 100644 --- a/apps/journal/app/lib/activities.server.ts +++ b/apps/journal/app/lib/activities.server.ts @@ -119,6 +119,18 @@ export async function createActivity(ownerId: string, input: ActivityInput) { await enqueueActivityDeliveries(ownerId, id, "create"); } + // Activities arrive without surface waytags (bare GPX); kick off the async + // Overpass backfill (route-surface-breakdown Path 2). Skipped for synthetic + // demo content. Best-effort — never blocks creation. + if (input.gpx && !input.synthetic) { + await enqueueOptional( + "surface-backfill", + { kind: "activity", id }, + { source: "createActivity" }, + { singletonKey: `surface:activity:${id}` }, + ); + } + return id; } diff --git a/apps/journal/app/lib/boss.server.ts b/apps/journal/app/lib/boss.server.ts index 94970a4..b1b7880 100644 --- a/apps/journal/app/lib/boss.server.ts +++ b/apps/journal/app/lib/boss.server.ts @@ -24,6 +24,8 @@ export interface BossSendOptions { retryLimit?: number; retryBackoff?: boolean; retryDelay?: number; + /** pg-boss dedup: collapses enqueues sharing a key into one active job. */ + singletonKey?: string; } interface BossLike { diff --git a/apps/journal/app/lib/overpass-ways.server.ts b/apps/journal/app/lib/overpass-ways.server.ts new file mode 100644 index 0000000..5ef404c --- /dev/null +++ b/apps/journal/app/lib/overpass-ways.server.ts @@ -0,0 +1,84 @@ +// Server-side Overpass client for the surface backfill: fetches highway ways +// (with their `surface` tag + geometry) in a bbox so a route/activity can be +// map-matched. Runs only in the backfill job, never in a request path. +// Privacy: this sends the route's bounding box to the upstream Overpass — see +// route-surface-breakdown design §D5; `OVERPASS_URLS` can point at a +// self-hosted instance to keep it in-house. + +export interface OverpassWay { + highway?: string; + surface?: string; + /** Polyline nodes, in order. */ + geometry: Array<{ lat: number; lon: number }>; +} + +export interface Bbox { + south: number; + west: number; + north: number; + east: number; +} + +const DEFAULT_UPSTREAMS = [ + "https://lz4.overpass-api.de/api/interpreter", + "https://overpass-api.de/api/interpreter", +]; + +function upstreams(): string[] { + const env = process.env.OVERPASS_URLS ?? process.env.OVERPASS_URL; + return env ? env.split(",").map((s) => s.trim()).filter(Boolean) : DEFAULT_UPSTREAMS; +} + +// Skip absurdly large bboxes — Overpass would time out / return too much, and a +// huge box means the route is so long that per-segment matching is unreliable +// anyway. ~0.25 deg² (roughly a 50 km square at mid latitudes). +const MAX_BBOX_DEG2 = 0.25; +const USER_AGENT = "trails.cool Journal (https://trails.cool; legal@trails.cool)"; + +/** + * Fetch highway ways within `bbox`. Returns `[]` for an empty/oversized bbox + * (caller treats that as "no data, don't store"); throws if every upstream + * fails so the job retries. + */ +export async function fetchWaysInBbox( + bbox: Bbox, + opts: { timeoutMs?: number } = {}, +): Promise { + const area = Math.abs(bbox.north - bbox.south) * Math.abs(bbox.east - bbox.west); + if (!(area > 0) || area > MAX_BBOX_DEG2) return []; + + const query = `[out:json][timeout:25];way["highway"](${bbox.south},${bbox.west},${bbox.north},${bbox.east});out tags geom;`; + const timeoutMs = opts.timeoutMs ?? 30_000; + + let lastError: unknown; + for (const url of upstreams()) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const resp = await fetch(url, { + method: "POST", + headers: { "Content-Type": "text/plain", "User-Agent": USER_AGENT }, + body: query, + signal: controller.signal, + }); + if (!resp.ok) { + lastError = new Error(`overpass ${resp.status}`); + continue; + } + const json = (await resp.json()) as { + elements?: Array<{ type: string; tags?: Record; geometry?: Array<{ lat: number; lon: number }> }>; + }; + const ways: OverpassWay[] = []; + for (const el of json.elements ?? []) { + if (el.type !== "way" || !el.geometry || el.geometry.length < 2) continue; + ways.push({ highway: el.tags?.highway, surface: el.tags?.surface, geometry: el.geometry }); + } + return ways; + } catch (e) { + lastError = e; + } finally { + clearTimeout(timer); + } + } + throw new Error(`overpass: all upstreams failed (${(lastError as Error)?.message ?? "unknown"})`); +} diff --git a/apps/journal/app/lib/surface-match.server.test.ts b/apps/journal/app/lib/surface-match.server.test.ts new file mode 100644 index 0000000..af0f195 --- /dev/null +++ b/apps/journal/app/lib/surface-match.server.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect } from "vitest"; +import { matchSurfaces } from "./surface-match.server.ts"; +import type { OverpassWay } from "./overpass-ways.server.ts"; + +// Two parallel ways: one along lat 0 (asphalt road), one along lat 1 (gravel track). +const ways: OverpassWay[] = [ + { + highway: "residential", + surface: "asphalt", + geometry: [ + { lat: 0, lon: 0 }, + { lat: 0, lon: 10 }, + ], + }, + { + highway: "track", + surface: "gravel", + geometry: [ + { lat: 1, lon: 0 }, + { lat: 1, lon: 10 }, + ], + }, +]; + +describe("matchSurfaces", () => { + it("assigns a segment near the lat-0 way its asphalt/residential tags", () => { + const { surfaces, highways } = matchSurfaces([[1, 0.1], [2, 0.1]], ways); + expect(surfaces).toEqual(["asphalt"]); + expect(highways).toEqual(["residential"]); + }); + + it("assigns a segment near the lat-1 way its gravel/track tags", () => { + const { surfaces, highways } = matchSurfaces([[1, 0.9], [2, 0.9]], ways); + expect(surfaces).toEqual(["gravel"]); + expect(highways).toEqual(["track"]); + }); + + it("returns unknown when there are no candidate ways", () => { + const { surfaces, highways } = matchSurfaces([[0, 0], [1, 0]], []); + expect(surfaces).toEqual(["unknown"]); + expect(highways).toEqual(["unknown"]); + }); + + it("buckets a missing surface tag as unknown but keeps the highway", () => { + const untagged: OverpassWay[] = [{ highway: "path", geometry: [{ lat: 0, lon: 0 }, { lat: 0, lon: 10 }] }]; + const { surfaces, highways } = matchSurfaces([[1, 0], [2, 0]], untagged); + expect(surfaces).toEqual(["unknown"]); + expect(highways).toEqual(["path"]); + }); +}); diff --git a/apps/journal/app/lib/surface-match.server.ts b/apps/journal/app/lib/surface-match.server.ts new file mode 100644 index 0000000..6d17a8d --- /dev/null +++ b/apps/journal/app/lib/surface-match.server.ts @@ -0,0 +1,59 @@ +import type { OverpassWay } from "./overpass-ways.server.ts"; + +/** Squared distance from point P to segment AB, in (degree) units — only used + * for *comparing* candidates, so the planar approximation is fine at the scale + * of a route bbox. */ +function distToSegmentSq( + px: number, py: number, + ax: number, ay: number, + bx: number, by: number, +): number { + const dx = bx - ax; + const dy = by - ay; + const lenSq = dx * dx + dy * dy; + let t = lenSq > 0 ? ((px - ax) * dx + (py - ay) * dy) / lenSq : 0; + t = Math.max(0, Math.min(1, t)); + const cx = ax + t * dx; + const cy = ay + t * dy; + return (px - cx) ** 2 + (py - cy) ** 2; +} + +/** + * Map-match each route segment to the nearest OSM way and read its + * surface/highway. `coords` are `[lon, lat, ...]`; returns `surfaces[i]` / + * `highways[i]` for the segment from `coords[i]` to `coords[i+1]` + * (length `coords.length - 1`). Segments with no candidate way → `"unknown"`. + * Feed the result through `computeSurfaceBreakdown` to get the distribution. + */ +export function matchSurfaces( + coords: ReadonlyArray, + ways: readonly OverpassWay[], +): { surfaces: string[]; highways: string[] } { + const surfaces: string[] = []; + const highways: string[] = []; + + for (let i = 0; i < coords.length - 1; i++) { + const a = coords[i]!; + const b = coords[i + 1]!; + const mx = (a[0]! + b[0]!) / 2; // lon + const my = (a[1]! + b[1]!) / 2; // lat + + let best: OverpassWay | null = null; + let bestDist = Infinity; + for (const way of ways) { + const g = way.geometry; + for (let j = 0; j < g.length - 1; j++) { + const d = distToSegmentSq(mx, my, g[j]!.lon, g[j]!.lat, g[j + 1]!.lon, g[j + 1]!.lat); + if (d < bestDist) { + bestDist = d; + best = way; + } + } + } + + surfaces.push(best?.surface ?? "unknown"); + highways.push(best?.highway ?? "unknown"); + } + + return { surfaces, highways }; +} diff --git a/apps/journal/app/routes/activities.$id.server.ts b/apps/journal/app/routes/activities.$id.server.ts index a39ef32..6c645ea 100644 --- a/apps/journal/app/routes/activities.$id.server.ts +++ b/apps/journal/app/routes/activities.$id.server.ts @@ -16,6 +16,7 @@ import { requireOwnedActivity, requireOwnedRoute } from "~/lib/ownership.server" import { parseGpxAsync, movingTime, elevationSeries } from "@trails-cool/gpx"; import type { ElevationSample } from "@trails-cool/gpx"; import { logger } from "~/lib/logger.server"; +import { enqueueOptional } from "~/lib/boss.server"; import type { Visibility } from "@trails-cool/db/schema/journal"; const VISIBILITY_VALUES = new Set(["private", "unlisted", "public"]); @@ -39,6 +40,19 @@ export async function loadActivityDetail(request: Request, id: string | undefine const userRoutes = isOwner && user ? await listRoutes(user.id) : []; + // Lazily backfill the surface breakdown for the owner's own activities that + // have geometry but no breakdown yet (route-surface-breakdown Path 2). The + // SSE event from the job fills the bars in live. Owner-gated so a random + // viewer can't trigger an Overpass lookup; idempotent via singletonKey. + if (isOwner && activity.geojson && !activity.surfaceBreakdown) { + await enqueueOptional( + "surface-backfill", + { kind: "activity", id: activity.id }, + { source: "activity-detail" }, + { singletonKey: `surface:activity:${activity.id}` }, + ); + } + // Moving time + the elevation series are both derived from the GPX (one // parse). Detail-page only — too costly to parse per row in list views. let movingTimeSec: number | null = null; diff --git a/apps/journal/app/routes/activities.$id.tsx b/apps/journal/app/routes/activities.$id.tsx index 86cb03f..b28bab0 100644 --- a/apps/journal/app/routes/activities.$id.tsx +++ b/apps/journal/app/routes/activities.$id.tsx @@ -8,6 +8,7 @@ import { SportBadge } from "~/components/SportBadge"; import { StatRow } from "~/components/StatRow"; import { ElevationProfile } from "~/components/ElevationProfile"; import { SurfaceBreakdown } from "~/components/SurfaceBreakdown"; +import { useSurfaceBackfillUpdates } from "~/hooks/useSurfaceBackfill"; import { activityStatItems } from "~/lib/stats"; import { loadActivityDetail, activityDetailAction } from "./activities.$id.server"; import { @@ -65,6 +66,9 @@ export default function ActivityDetailPage({ loaderData }: Route.ComponentProps) const { activity, isOwner, routes } = loaderData; const { t } = useTranslation("journal"); + // Live-update when the async surface backfill lands (owner-only). + useSurfaceBackfillUpdates("activity", activity.id, isOwner && !activity.surfaceBreakdown); + // Elevation profile ↔ map sync via a shared "active" sample index. const elevation = activity.elevation; const [activeIndex, setActiveIndex] = useState(null); diff --git a/apps/journal/app/routes/legal.privacy.tsx b/apps/journal/app/routes/legal.privacy.tsx index 21518be..f2b10fe 100644 --- a/apps/journal/app/routes/legal.privacy.tsx +++ b/apps/journal/app/routes/legal.privacy.tsx @@ -253,12 +253,16 @@ export default function PrivacyPage() { .
  • - Overpass API (POI-Daten) – POI-Abfragen laufen - serverseitig über unsere eigene Route /api/overpass. - Der Upstream-Dienst sieht nur unsere Server-IP, nicht die Ihrer - Nutzer:innen. Aktueller Upstream: overpass.private.coffee - , der ohne Query-Logs arbeitet. Eine selbst gehostete Instanz ist - geplant. + Overpass API (POI- und Wegbelag-Daten) – + POI-Abfragen laufen serverseitig über unsere eigene Route{" "} + /api/overpass. Für die Oberflächen-/Wegtyp-Auswertung + importierter Aktivitäten und Routen fragt ein Hintergrund-Job das + Begrenzungsrechteck (Bounding Box) der Route bei Overpass ab, um sie + den OpenStreetMap-Wegen zuzuordnen. In beiden Fällen sieht der + Upstream nur unsere Server-IP, nicht die Ihrer Nutzer:innen. + Aktueller Upstream: overpass.private.coffee bzw. die + öffentlichen Overpass-Instanzen, die ohne Query-Logs arbeiten. Eine + selbst gehostete Instanz ist geplant.
  • Föderation (ActivityPub) – Wenn Ihr Profil auf{" "} @@ -334,8 +338,11 @@ export default function PrivacyPage() { English. Third parties and what they receive: Sentry (error details, no IPs/cookies); OpenStreetMap tile servers (your IP and user-agent, directly from your browser, to load map - tiles); Overpass (via our server-side proxy, so upstream only sees - our server); BRouter (self-hosted, no third party involved); SMTP + tiles); Overpass (via our server, so upstream only sees our server — + for POI lookups and, for the surface/waytype breakdown of imported + activities and routes, a background job that sends the route’s + bounding box to match it against OpenStreetMap ways); BRouter + (self-hosted, no third party involved); SMTP provider (your email address for magic link / welcome mail); Wahoo (only when you opt in: OAuth tokens for sync, plus route geometry/name/description when you click “Send to diff --git a/apps/journal/app/routes/routes.$id.server.ts b/apps/journal/app/routes/routes.$id.server.ts index 749d1be..27f0b3b 100644 --- a/apps/journal/app/routes/routes.$id.server.ts +++ b/apps/journal/app/routes/routes.$id.server.ts @@ -11,6 +11,7 @@ import { syncPushes } from "@trails-cool/db/schema/journal"; import { getService } from "~/lib/connected-services"; import { computeDays, parseGpxAsync, elevationSeries } from "@trails-cool/gpx"; import type { ElevationSample } from "@trails-cool/gpx"; +import { enqueueOptional } from "~/lib/boss.server"; export async function loadRouteDetail(request: Request, id: string | undefined) { const routeId = id ?? ""; @@ -30,6 +31,18 @@ export async function loadRouteDetail(request: Request, id: string | undefined) throw data({ error: "Route not found" }, { status: 404 }); } + // Lazy surface backfill for the owner's own routes that lack a breakdown + // (non-Planner routes — Planner saves provide it synchronously). Owner-gated; + // idempotent via singletonKey. The SSE event fills the bars in live. + if (isOwner && routeWithGeojson?.geojson && !route.surfaceBreakdown) { + await enqueueOptional( + "surface-backfill", + { kind: "route", id: route.id }, + { source: "route-detail" }, + { singletonKey: `surface:route:${route.id}` }, + ); + } + // Parse GPX once for day stats and waypoint POI data let dayStats: Array<{ dayNumber: number; startName?: string; endName?: string; distance: number; ascent: number; descent: number }> = []; let waypoints: Array<{ lat: number; lon: number; name?: string; isDayBreak?: boolean; note?: string; osmId?: number; poiTags?: Record }> = []; diff --git a/apps/journal/app/routes/routes.$id.tsx b/apps/journal/app/routes/routes.$id.tsx index fdeae34..b45a02d 100644 --- a/apps/journal/app/routes/routes.$id.tsx +++ b/apps/journal/app/routes/routes.$id.tsx @@ -7,6 +7,7 @@ import { ClientMap } from "~/components/ClientMap"; import { StatRow } from "~/components/StatRow"; import { ElevationProfile } from "~/components/ElevationProfile"; import { SurfaceBreakdown } from "~/components/SurfaceBreakdown"; +import { useSurfaceBackfillUpdates } from "~/hooks/useSurfaceBackfill"; import { activityStatItems } from "~/lib/stats"; import { loadRouteDetail, routeDetailAction } from "./routes.$id.server"; @@ -47,6 +48,10 @@ export function meta({ data: loaderData }: Route.MetaArgs) { export default function RouteDetailPage({ loaderData }: Route.ComponentProps) { const { route, dayStats, waypoints, versions, isOwner, wahooPush } = loaderData; const { t, i18n } = useTranslation("journal"); + + // Live-update when the async surface backfill lands (owner-only). + useSurfaceBackfillUpdates("route", route.id, isOwner && !route.surfaceBreakdown); + const [editLoading, setEditLoading] = useState(false); const [highlightedDay, setHighlightedDay] = useState(null); diff --git a/apps/journal/server.ts b/apps/journal/server.ts index 02ca41d..839e3e3 100644 --- a/apps/journal/server.ts +++ b/apps/journal/server.ts @@ -178,7 +178,8 @@ server.listen(port, async () => { const { sendWelcomeEmailJob } = await import("./app/jobs/send-welcome-email.ts"); const { consumedJtiSweepJob } = await import("./app/jobs/consumed-jti-sweep.ts"); const { garminImportActivityJob } = await import("./app/jobs/garmin-import-activity.ts"); - jobs.push(notificationsFanoutJob, notificationsPurgeJob, komootBulkImportJob, importBatchesSweepJob, sendWelcomeEmailJob, consumedJtiSweepJob, garminImportActivityJob); + const { surfaceBackfillJob } = await import("./app/jobs/surface-backfill.ts"); + jobs.push(notificationsFanoutJob, notificationsPurgeJob, komootBulkImportJob, importBatchesSweepJob, sendWelcomeEmailJob, consumedJtiSweepJob, garminImportActivityJob, surfaceBackfillJob); // Federation jobs — registered only when federation is on. if (process.env.FEDERATION_ENABLED === "true") { const { backfillUserKeypairsJob } = await import("./app/jobs/backfill-user-keypairs.ts"); diff --git a/openspec/changes/route-surface-breakdown/tasks.md b/openspec/changes/route-surface-breakdown/tasks.md index 55a0a52..2f1c036 100644 --- a/openspec/changes/route-surface-breakdown/tasks.md +++ b/openspec/changes/route-surface-breakdown/tasks.md @@ -21,21 +21,21 @@ - [x] 4.2 `SurfaceBreakdown` component (stacked bar per dimension, `map-core` palettes, legend category · % · km largest-first, unknown → "other", hidden when empty); mounted on `routes.$id.tsx` and `activities.$id.tsx`. - [x] 4.3 i18n `journal.surface.*` (surface/waytype labels, common categories, "other") en + de. -## 5. Path 2 — async backfill + SSE (PHASE 2 — follow-up PR) +## 5. Path 2 — async backfill + SSE (PHASE 2 — this PR) -- [ ] 5.1 Journal-side Overpass client: `way[highway]` in bbox + parse (mirror the planner's `buildQuery`/`parseResponse` + rate-limit handling; route via the `/api/overpass` proxy). -- [ ] 5.2 Map-match helper: nearest OSM way per route segment → surface/highway; unmatched → unknown. Unit-test the matcher on a small fixture. -- [ ] 5.3 Typed `surface-backfill` pg-boss job (`{ kind, id }`): load geometry → skip if breakdown exists → Overpass → map-match → store. Idempotent, retry-safe, best-effort. -- [ ] 5.4 Enqueue the job after GPX import (Komoot/Garmin) and manual upload; add a one-off sweep entrypoint for pre-existing rows. -- [ ] 5.5 Emit a `surface_breakdown` SSE event (`{ kind, id }`) on completion; detail page subscribes (EventSource hook) and re-fetches the breakdown when it matches; optional "computing…" state. +- [x] 5.1 Journal-side Overpass client (`overpass-ways.server.ts`): `way[highway]` in bbox + `out tags geom`, configurable `OVERPASS_URLS` upstreams, timeout + oversized-bbox guard. +- [x] 5.2 Map-matcher (`surface-match.server.ts`, pure): nearest OSM way per segment → surface/highway; unmatched → unknown. Unit-tested. +- [x] 5.3 Typed `surface-backfill` pg-boss job: load geom → skip if breakdown exists → Overpass → match → `computeSurfaceBreakdown` → store. Idempotent, retry-safe, best-effort; registered in `server.ts`. +- [x] 5.4 Enqueued from `createActivity` (covers Komoot/Garmin imports + manual uploads) and owner-on-open in the route + activity detail loaders (covers non-Planner routes + pre-existing rows), deduped via `singletonKey`. (A dedicated sweep CLI is unneeded — on-open covers existing rows lazily.) +- [x] 5.5 Emits a `surface_breakdown` SSE event (`{ kind, id }`) to the owner; detail pages subscribe (`useSurfaceBackfillUpdates`) and `revalidate()` when it matches. ## 6. Privacy -- [ ] 6.1 Document the backfill's Overpass lookup in the privacy manifest; route via the proxy (self-hostable Overpass is the long-term answer — see `docs/ideas/self-host-overpass`). +- [x] 6.1 Documented the backfill's Overpass bbox lookup in the privacy manifest (DE + EN); `OVERPASS_URLS` can point at a self-hosted instance. ## 7. Tests & checks -- [x] 7.1 Unit: `computeSurfaceBreakdown` weighting + unknown bucket (map-matcher unit is Phase 2). +- [x] 7.1 Unit: `computeSurfaceBreakdown` weighting + unknown bucket; `matchSurfaces` nearest-way + unknown bucket. - [x] 7.2 Component (jsdom): bars/proportions, largest-first, hidden when empty/null. -- [ ] 7.3 E2E: a route/activity with a stored breakdown shows the bars — **Phase 2** (needs a seed path; deferred to land with the backfill that creates the data). Phase 1 render verified in the browser + component test. -- [x] 7.4 Phase 1: typecheck + lint + unit (map-core 36, journal 326) green; verified in the browser. +- [x] 7.3 Job test (mocked Overpass/db/events): stores + emits; skips when breakdown exists or no ways. (A full live e2e is intentionally skipped — external Overpass + SSE timing are flaky; rendering is covered by the component test + browser, the job by its mocked test.) +- [x] 7.4 typecheck + lint + unit (map-core 36, journal 333) green; Phase 1 verified in the browser.