Task group 1 of federation-hardening. Fedify was configured with `InProcessMessageQueue`, so every queued outbound delivery, its pending retry state, and inbox processing task was lost on a container restart (routine here: deploys, OOM history) — directly contradicting the social-federation spec's promise that fan-out survives a deploy. - `federation-queue.server.ts`: `PgBossMessageQueue` implementing Fedify's `MessageQueue` over the pg-boss instance the journal already runs, mirroring the `PostgresKvStore` adapter. `nativeRetrial = false` keeps Fedify the retry-policy owner; pg-boss supplies durability + delayed jobs (delay → whole-second `startAfter`, `retryLimit: 0`). - Swap it in for `InProcessMessageQueue` in `federation.server.ts`; document the two intentional queueing layers (our fan-out jobs feed Fedify; Fedify's sends now durable underneath). - `server.ts`: create the durable queue at startup when federation is on. - Broaden the structural `BossLike` in `boss.server.ts` with the work/offWork/createQueue/getQueue methods the adapter needs. - Tests: enqueue maps retry/delay correctly, consume roundtrip, restart durability (fresh listener drains a prior instance's backlog), abort stops the worker, depth reports ready vs delayed. Verified: journal typecheck + lint clean, 7/7 new unit tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
109 lines
3.9 KiB
TypeScript
109 lines
3.9 KiB
TypeScript
// Process-level pg-boss singleton. server.ts initializes the boss + starts
|
|
// the worker; feature code (e.g., activities.server.ts) calls `getBoss()`
|
|
// to enqueue jobs against the same instance. The singleton's lifecycle
|
|
// is bound to the Node process — startWorker calls boss.start(); the
|
|
// SIGTERM handler stops it.
|
|
//
|
|
// The instance lives on globalThis, NOT in a module-local variable. In
|
|
// production there are TWO copies of this module in the process:
|
|
// server.ts imports the TypeScript source directly (node
|
|
// --experimental-strip-types), while route handlers live in the
|
|
// bundled build/server/index.js with its own module instance. A
|
|
// module-local `_boss` set by server.ts is invisible to the bundle, so
|
|
// every request-path enqueue (notifications fan-out, federation
|
|
// deliveries, komoot imports) silently failed in production with
|
|
// "pg-boss not initialized". Symbol.for() gives both copies the same
|
|
// global registry key.
|
|
|
|
import { logger } from "./logger.server.ts";
|
|
import type { JobName, JobPayloads } from "../jobs/payloads.ts";
|
|
|
|
// Structurally typed so we don't have to pull pg-boss into the journal
|
|
// app's dep graph just for the typedef. Kept to the subset feature code
|
|
// actually uses: `send` for enqueues, plus `work`/`offWork`/`createQueue`/
|
|
// `getQueue` for the Fedify message-queue adapter (federation-queue.server.ts).
|
|
export interface BossSendOptions {
|
|
retryLimit?: number;
|
|
retryBackoff?: boolean;
|
|
retryDelay?: number;
|
|
/** Seconds to defer the job (pg-boss `startAfter`). */
|
|
startAfter?: number;
|
|
/** pg-boss dedup: collapses enqueues sharing a key into one active job. */
|
|
singletonKey?: string;
|
|
}
|
|
|
|
/** A pg-boss job as our handlers see it (subset of pg-boss's `Job`). */
|
|
export interface BossJob<T = unknown> {
|
|
id: string;
|
|
name: string;
|
|
data: T;
|
|
}
|
|
|
|
/** Queue depth counters (subset of pg-boss's `QueueResult`). */
|
|
export interface BossQueueDepth {
|
|
queuedCount: number;
|
|
readyCount: number;
|
|
deferredCount: number;
|
|
}
|
|
|
|
interface BossLike {
|
|
send(queueName: string, data: unknown, options?: BossSendOptions): Promise<string | null>;
|
|
work(queueName: string, handler: (jobs: BossJob[]) => Promise<unknown>): Promise<string>;
|
|
offWork(queueName: string): Promise<void>;
|
|
createQueue(queueName: string, options?: unknown): Promise<void>;
|
|
getQueue(queueName: string): Promise<BossQueueDepth | null>;
|
|
}
|
|
|
|
const BOSS_KEY = Symbol.for("trails-cool.journal.pg-boss");
|
|
|
|
type BossGlobal = { [BOSS_KEY]?: BossLike | null };
|
|
|
|
/** Set by server.ts once the boss is created + started. */
|
|
export function setBoss(boss: BossLike | null): void {
|
|
(globalThis as BossGlobal)[BOSS_KEY] = boss;
|
|
}
|
|
|
|
/**
|
|
* Get the started pg-boss instance. Throws if called before
|
|
* server.ts has initialized it (i.e., outside a running Journal
|
|
* server context).
|
|
*/
|
|
export function getBoss(): BossLike {
|
|
const boss = (globalThis as BossGlobal)[BOSS_KEY];
|
|
if (!boss) {
|
|
throw new Error("pg-boss not initialized — getBoss called before server bootstrap");
|
|
}
|
|
return boss;
|
|
}
|
|
|
|
/**
|
|
* Enqueue a job. The queue name must be a key of JobPayloads and the
|
|
* payload must match its declared shape. Throws if the queue is down —
|
|
* use this when the caller's correctness depends on the job existing
|
|
* (e.g. a batch row that would otherwise wait forever).
|
|
*/
|
|
export async function enqueue<K extends JobName>(
|
|
queue: K,
|
|
data: JobPayloads[K],
|
|
options?: BossSendOptions,
|
|
): Promise<void> {
|
|
await getBoss().send(queue, data, options);
|
|
}
|
|
|
|
/**
|
|
* Best-effort enqueue: log + swallow errors so a downstream queue
|
|
* outage doesn't fail the user-visible request that triggered the
|
|
* fan-out. Use this for "fire and forget" notifications work.
|
|
*/
|
|
export async function enqueueOptional<K extends JobName>(
|
|
queue: K,
|
|
data: JobPayloads[K],
|
|
ctx: Record<string, unknown> = {},
|
|
options?: BossSendOptions,
|
|
): Promise<void> {
|
|
try {
|
|
await enqueue(queue, data, options);
|
|
} catch (err) {
|
|
logger.warn({ err, queue, ...ctx }, "boss.send failed; continuing");
|
|
}
|
|
}
|