trails/apps/journal/app/lib/federation-ingest.integration.test.ts
Ullrich Schäfer b96bef91a9 feat(journal): outbox-poll ingestion of remote trails activities (§7)
Tasks 7.1–7.3, 7.5 (+7.4 pacing; Retry-After-duration backoff noted as
remaining). Resolves the activities.owner_id open question.

Schema (design decision from the open question):
- activities.owner_id nullable + check constraint enforcing exactly
  one of (owner_id, remote_actor_iri) — the follows pattern again.
- remote_published_at carries the origin's publish time for the §8
  feed sort; index on remote_actor_iri for the feed join.
- Compiler-audited fallout: notification fan-out, the Note object
  dispatcher, and the activity detail loader explicitly skip/404
  remote rows (their canonical page is the origin; feed links there).
  Every other surface joins users on owner_id and excludes remote rows
  structurally.

Ingestion:
- federation-ingest.server.ts: parseOutboxItem targets exactly our own
  outgoing Create(Note) shape (PropertyValue stats, first-paragraph
  name, audience from to/cc); unknown items skipped, never fatal.
- ingestRemoteActivities: replay-safe via unique remote_origin_iri,
  conflict-streak early exit (outboxes are newest-first).
- pollRemoteActor: signed fetch (Authorized Fetch via a local
  follower's key), outbox resolution via remote_actors cache with
  actor-doc fallback (which refreshes the cache), 1 req/5 s per-host
  pacing, last_polled_at stamping. Network + pacing injectable.

Jobs: poll-remote-actor (per-actor; the §4 Accept listener already
enqueues it as the first poll) + poll-remote-outboxes (5-min cron
sweep over accepted remote follows not polled within the hour).

Tests: parse-shape units; integration suite for ingestion provenance,
audience→visibility mirroring, replay safety, the DB author invariant,
poll flow (actor-doc → collection → page), signer requirement, and
due-polling selection.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-07 12:27:10 +02:00

167 lines
5.6 KiB
TypeScript

import { describe, it, expect, beforeAll, afterEach } from "vitest";
import { eq, like } from "drizzle-orm";
import { randomUUID } from "node:crypto";
import { getDb } from "./db.ts";
import { users, follows, activities, remoteActors } from "@trails-cool/db/schema/journal";
import {
ingestRemoteActivities,
pollRemoteActor,
listActorsDuePolling,
type ParsedRemoteActivity,
} from "./federation-ingest.server.ts";
// Opt-in: real Postgres; network injected. Same convention as the
// other *.integration.test.ts files.
const runIntegration = process.env.FEDERATION_INTEGRATION === "1";
const ACTOR = `https://poll-target.example/users/alice-${Date.now()}`;
const createdUserIds: string[] = [];
async function makeFollowerOf(actorIri: string) {
const db = getDb();
const id = randomUUID();
await db.insert(users).values({
id,
email: `poll-${id}@example.test`,
username: `poll-${id.slice(0, 8)}`,
domain: "test.local",
profileVisibility: "public",
});
createdUserIds.push(id);
await db.insert(follows).values({
id: randomUUID(),
followerId: id,
followedActorIri: actorIri,
followedUserId: null,
acceptedAt: new Date(),
});
return id;
}
function parsed(n: number, audience: "public" | "followers-only" = "public"): ParsedRemoteActivity {
return {
originIri: `${ACTOR}/activities/${n}`,
name: `Remote activity ${n}`,
distance: 1000 * n,
elevationGain: null,
duration: null,
publishedAt: new Date(Date.now() - n * 60_000),
audience,
};
}
describe.runIf(runIntegration)("outbox-poll ingestion (integration)", () => {
beforeAll(() => {
process.env.ORIGIN ??= "http://localhost:3000";
});
afterEach(async () => {
const db = getDb();
await db.delete(activities).where(like(activities.remoteOriginIri, `${ACTOR}%`));
await db.delete(remoteActors).where(eq(remoteActors.actorIri, ACTOR));
for (const id of createdUserIds.splice(0)) {
await db.delete(follows).where(eq(follows.followerId, id));
await db.delete(users).where(eq(users.id, id));
}
});
it("inserts remote rows with provenance and audience-mirrored visibility", async () => {
const inserted = await ingestRemoteActivities(ACTOR, [parsed(1), parsed(2, "followers-only")]);
expect(inserted).toBe(2);
const db = getDb();
const rows = await db
.select()
.from(activities)
.where(like(activities.remoteOriginIri, `${ACTOR}%`));
expect(rows).toHaveLength(2);
for (const row of rows) {
expect(row.ownerId).toBeNull();
expect(row.remoteActorIri).toBe(ACTOR);
expect(row.remotePublishedAt).not.toBeNull();
}
const byIri = new Map(rows.map((r) => [r.remoteOriginIri, r]));
expect(byIri.get(`${ACTOR}/activities/1`)?.visibility).toBe("public");
expect(byIri.get(`${ACTOR}/activities/2`)?.visibility).toBe("private");
expect(byIri.get(`${ACTOR}/activities/2`)?.audience).toBe("followers-only");
});
it("is replay-safe: re-ingesting inserts nothing", async () => {
await ingestRemoteActivities(ACTOR, [parsed(1), parsed(2)]);
const again = await ingestRemoteActivities(ACTOR, [parsed(1), parsed(2)]);
expect(again).toBe(0);
});
it("database enforces exactly-one-author invariant", async () => {
const db = getDb();
await expect(
db.insert(activities).values({
id: randomUUID(),
ownerId: null,
remoteActorIri: null,
name: "orphan",
}),
).rejects.toThrow();
});
it("pollRemoteActor resolves the outbox via the actor doc, ingests the first page, stamps last_polled_at", async () => {
await makeFollowerOf(ACTOR);
const fetched: string[] = [];
const result = await pollRemoteActor(ACTOR, {
async pace() {},
async fetchJson(url) {
fetched.push(url);
if (url === ACTOR) {
return { outbox: `${ACTOR}/outbox`, inbox: `${ACTOR}/inbox`, preferredUsername: "alice", name: "Alice" };
}
if (url === `${ACTOR}/outbox`) {
return { type: "OrderedCollection", totalItems: 1, first: `${ACTOR}/outbox?cursor=0` };
}
return {
type: "OrderedCollectionPage",
orderedItems: [
{
type: "Create",
object: {
type: "Note",
id: `${ACTOR}/activities/77`,
content: "<p>Polled ride</p>",
published: "2026-06-05T10:00:00Z",
to: "as:Public",
},
},
],
};
},
});
expect(result).toEqual({ inserted: 1 });
expect(fetched).toEqual([ACTOR, `${ACTOR}/outbox`, `${ACTOR}/outbox?cursor=0`]);
const db = getDb();
const [cached] = await db.select().from(remoteActors).where(eq(remoteActors.actorIri, ACTOR));
expect(cached?.outboxUrl).toBe(`${ACTOR}/outbox`);
expect(cached?.lastPolledAt).not.toBeNull();
});
it("skips actors without an accepted local follower", async () => {
const result = await pollRemoteActor(ACTOR, {
async pace() {},
async fetchJson() {
throw new Error("must not fetch");
},
});
expect(result).toEqual({ skipped: "no accepted local follower" });
});
it("listActorsDuePolling returns followed actors not polled within the hour", async () => {
await makeFollowerOf(ACTOR);
expect(await listActorsDuePolling()).toContain(ACTOR);
const db = getDb();
await db.insert(remoteActors).values({ actorIri: ACTOR, lastPolledAt: new Date() }).onConflictDoUpdate({
target: remoteActors.actorIri,
set: { lastPolledAt: new Date() },
});
expect(await listActorsDuePolling()).not.toContain(ACTOR);
});
});