trails/apps/journal/app/lib/komoot-bulk-import.server.ts
Ullrich Schäfer 5ff7af8a81
Add background bulk import for Komoot
Replace the per-tour import UI with a fire-and-forget background job:

- Add `import_batches` DB table tracking status, found/imported/duplicate counts
- Add `runKomootBulkImport` server function that pages all Komoot tours,
  fetches GPX, creates activities, and deduplicates via sync_imports
- Add `komoot-bulk-import` pg-boss job registered at server startup
- Add POST /api/sync/komoot/import to enqueue the job
- Add GET /api/sync/komoot/import-status to return the latest batch
- Replace the Komoot import page with a progress UI that polls every 2s
  while a batch is running and shows found/imported/skipped counts

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-23 18:19:10 +02:00

114 lines
3.6 KiB
TypeScript

import { eq } from "drizzle-orm";
import { getDb } from "./db.ts";
import { importBatches } from "@trails-cool/db/schema/journal";
import { fetchKomootTours, fetchKomootTourGpx } from "./komoot.server.ts";
import { isAlreadyImported, recordImport } from "./sync/imports.server.ts";
import { createActivity } from "./activities.server.ts";
import { decrypt } from "./crypto.server.ts";
import { logger } from "./logger.server.ts";
type KomootCreds =
| { mode: "public"; komootUserId: string }
| { mode: "authenticated"; email: string; encryptedPassword: string; komootUserId: string };
function getBasicAuthToken(creds: KomootCreds): string | undefined {
if (creds.mode !== "authenticated") return undefined;
const password = decrypt(creds.encryptedPassword);
return Buffer.from(`${creds.email}:${password}`).toString("base64");
}
export async function runKomootBulkImport(
batchId: string,
userId: string,
creds: KomootCreds,
): Promise<void> {
const db = getDb();
const basicAuthToken = getBasicAuthToken(creds);
const komootUserId = creds.komootUserId;
await db
.update(importBatches)
.set({ status: "running" })
.where(eq(importBatches.id, batchId));
try {
let page = 1;
let hasMore = true;
let totalFound = 0;
let importedCount = 0;
let duplicateCount = 0;
while (hasMore) {
let result: Awaited<ReturnType<typeof fetchKomootTours>>;
try {
result = await fetchKomootTours(komootUserId, page, basicAuthToken);
} catch (err) {
logger.warn({ batchId, page, err }, "komoot bulk import: fetchKomootTours failed, stopping");
break;
}
totalFound += result.tours.length;
hasMore = page < result.totalPages;
page++;
await db
.update(importBatches)
.set({ totalFound })
.where(eq(importBatches.id, batchId));
for (const tour of result.tours) {
if (await isAlreadyImported(userId, "komoot", tour.id)) {
duplicateCount++;
await db
.update(importBatches)
.set({ duplicateCount })
.where(eq(importBatches.id, batchId));
continue;
}
let gpx: string | undefined;
try {
gpx = await fetchKomootTourGpx(tour.id, basicAuthToken);
} catch {
// No GPX — import metadata only
}
try {
const activityId = await createActivity(userId, {
name: tour.name,
gpx,
distance: tour.distance > 0 ? tour.distance : null,
duration: tour.duration > 0 ? tour.duration : null,
startedAt: tour.date ? new Date(tour.date) : null,
});
await recordImport(userId, "komoot", tour.id, activityId);
importedCount++;
} catch (err) {
logger.warn({ batchId, tourId: tour.id, err }, "komoot bulk import: failed to import tour");
}
await db
.update(importBatches)
.set({ importedCount, duplicateCount })
.where(eq(importBatches.id, batchId));
}
}
await db
.update(importBatches)
.set({ status: "completed", totalFound, importedCount, duplicateCount, completedAt: new Date() })
.where(eq(importBatches.id, batchId));
logger.info({ batchId, totalFound, importedCount, duplicateCount }, "komoot bulk import completed");
} catch (err) {
await db
.update(importBatches)
.set({
status: "failed",
errorMessage: err instanceof Error ? err.message : String(err),
completedAt: new Date(),
})
.where(eq(importBatches.id, batchId));
throw err;
}
}