Task group 4 of federation-hardening.
4.1 — FEDERATION.md at the repo root: actor discovery (WebFinger, actor,
NodeInfo), object/activity types with real JSON examples (Note, Create,
Delete, the narrow follow-graph inbox), addressing, HTTP-Signature
expectations, the two-layer dedup contract, durable delivery/retry
policy, and blocklist moderation semantics — precise enough for another
implementation to interoperate. Linked from README and docs/architecture.
4.2 — three prom-client metrics + a journal dashboard row:
- `federation_delivery_total{outcome}` — incremented in deliver-activity
(delivered/skipped/failed).
- `federation_inbox_dropped_total{reason}` — incremented at every inbox
drop (duplicate | blocked); this is the counter deferred from task 3.2.
- `federation_queue_depth` — gauge sampled at scrape time in
/api/metrics from PgBossMessageQueue.getDepth(); the restart-loss
regression detector.
Grafana journal.json gains a Federation row (delivery rate, queue depth,
inbox drops); the logs panels shift down to make room.
Verified: dashboard JSON valid; journal typecheck + lint clean; unit
suite 357 passing (route-template guard unaffected).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
134 lines
4.8 KiB
TypeScript
134 lines
4.8 KiB
TypeScript
import { defineJournalJob } from "./payloads.ts";
|
|
import { and, eq } from "drizzle-orm";
|
|
import { activities, users } from "@trails-cool/db/schema/journal";
|
|
import { getDb } from "../lib/db.ts";
|
|
import { getOrigin } from "../lib/config.server.ts";
|
|
import { getFederation } from "../lib/federation.server.ts";
|
|
import {
|
|
activityToCreate,
|
|
activityToDelete,
|
|
type FederatableActivity,
|
|
} from "../lib/federation-objects.server.ts";
|
|
import {
|
|
getCachedRemoteActor,
|
|
upsertRemoteActor,
|
|
type DeliveryPayload,
|
|
} from "../lib/federation-delivery.server.ts";
|
|
import { logger } from "../lib/logger.server.ts";
|
|
import { federationDeliveryTotal } from "../lib/metrics.server.ts";
|
|
|
|
/**
|
|
* Outbound pacing (spec 5.5): never exceed 1 request/second per remote
|
|
* host. In-process map is sufficient — pg-boss works this queue
|
|
* sequentially in the single journal process.
|
|
*/
|
|
const lastSendPerHost = new Map<string, number>();
|
|
const MIN_INTERVAL_MS = 1000;
|
|
|
|
async function paceHost(host: string): Promise<void> {
|
|
const last = lastSendPerHost.get(host) ?? 0;
|
|
const wait = last + MIN_INTERVAL_MS - Date.now();
|
|
if (wait > 0) await new Promise((r) => setTimeout(r, wait));
|
|
lastSendPerHost.set(host, Date.now());
|
|
}
|
|
|
|
/**
|
|
* Deliver one activity to one remote follower's inbox (spec 5.3/5.4).
|
|
* One job per (activity, recipient) so each delivery retries with
|
|
* exponential backoff independently (configured at enqueue time —
|
|
* see enqueueActivityDeliveries). A thrown error marks the attempt
|
|
* failed and pg-boss retries; exhausting the budget is the permanent
|
|
* failure, logged by the final catch.
|
|
*/
|
|
export const deliverActivityJob = defineJournalJob({
|
|
name: "deliver-activity",
|
|
expireInSeconds: 60,
|
|
async handler(jobs) {
|
|
for (const job of jobs) {
|
|
const p = job.data;
|
|
try {
|
|
const outcome = await deliverOne(p);
|
|
federationDeliveryTotal.inc({ outcome });
|
|
} catch (err) {
|
|
federationDeliveryTotal.inc({ outcome: "failed" });
|
|
logger.warn(
|
|
{ err, action: p.action, objectIri: p.objectIri, recipient: p.recipientActorIri },
|
|
"deliver-activity attempt failed (pg-boss will retry until budget exhausted)",
|
|
);
|
|
throw err;
|
|
}
|
|
}
|
|
},
|
|
});
|
|
|
|
async function deliverOne(p: DeliveryPayload): Promise<"delivered" | "skipped"> {
|
|
const federation = getFederation();
|
|
const ctx = federation.createContext(new URL(getOrigin()), undefined);
|
|
|
|
// Build the activity to send. For `create`, re-read the row at
|
|
// delivery time: if it was deleted or un-publicized since enqueue,
|
|
// skip rather than leak.
|
|
let activity;
|
|
if (p.action === "create") {
|
|
const db = getDb();
|
|
const [row] = await db
|
|
.select()
|
|
.from(activities)
|
|
.where(and(eq(activities.id, p.activityId!), eq(activities.visibility, "public")))
|
|
.limit(1);
|
|
if (!row) {
|
|
logger.info({ objectIri: p.objectIri }, "deliver-activity: activity gone or non-public; skipping");
|
|
return "skipped";
|
|
}
|
|
// Spec 9.3: flipping the profile to private stops federation — also
|
|
// for deliveries already enqueued when the flip happened.
|
|
const [owner] = await db
|
|
.select({ profileVisibility: users.profileVisibility })
|
|
.from(users)
|
|
.where(eq(users.username, p.ownerUsername))
|
|
.limit(1);
|
|
if (!owner || owner.profileVisibility !== "public") {
|
|
logger.info({ objectIri: p.objectIri }, "deliver-activity: owner no longer public; skipping");
|
|
return "skipped";
|
|
}
|
|
activity = activityToCreate(row as FederatableActivity, p.ownerUsername);
|
|
} else {
|
|
activity = activityToDelete(p.objectIri, p.ownerUsername);
|
|
}
|
|
|
|
// Resolve the recipient's inbox: cached remote_actors row first,
|
|
// actor-document fetch as fallback (which also primes the cache).
|
|
const recipientIri = new URL(p.recipientActorIri);
|
|
let inboxUrl: URL;
|
|
const cached = await getCachedRemoteActor(p.recipientActorIri);
|
|
if (cached?.inboxUrl) {
|
|
inboxUrl = new URL(cached.inboxUrl);
|
|
} else {
|
|
await paceHost(recipientIri.host);
|
|
const actor = await ctx.lookupObject(recipientIri);
|
|
const fetchedInbox =
|
|
actor != null && "inboxId" in actor ? (actor.inboxId as URL | null) : null;
|
|
if (!fetchedInbox) {
|
|
throw new Error(`deliver-activity: cannot resolve inbox for ${p.recipientActorIri}`);
|
|
}
|
|
inboxUrl = fetchedInbox;
|
|
await upsertRemoteActor({
|
|
actorIri: p.recipientActorIri,
|
|
inboxUrl: inboxUrl.href,
|
|
domain: recipientIri.host,
|
|
});
|
|
}
|
|
|
|
await paceHost(inboxUrl.host);
|
|
await ctx.sendActivity(
|
|
// Identifier and username are the same thing in our actor model.
|
|
{ identifier: p.ownerUsername },
|
|
{ id: recipientIri, inboxId: inboxUrl },
|
|
activity,
|
|
);
|
|
logger.info(
|
|
{ action: p.action, objectIri: p.objectIri, recipient: p.recipientActorIri },
|
|
"deliver-activity: delivered",
|
|
);
|
|
return "delivered";
|
|
}
|