@devries-cameron20 / t3code-1 / commits / 80455376f0
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

perf(orchestration): thread snapshots send each long ID once (#24643)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Theo Browne committed Oct 24, 2026 at 20:02 UTC 80455376f01f19c10a06dc08b0629486c017bf79
14 files changed +369 -54
apps/server/src/orchestration-v2/ThreadStream.ts
+4
@@ -41,6 +41,7 @@ export function boundedSnapshotResponseFields(input: {
41 readonly bounded: BoundedProjectionResult;
42 readonly compactTurnItems: boolean;
43 readonly dedupeBodies: boolean;
44 + readonly compactIds: boolean;
45 }) {
46 const { bounded } = input;
47 // Dedupe keeps shared items one object, which the compaction's identity check needs.
@@ -54,6 +55,7 @@ export function boundedSnapshotResponseFields(input: {
55 hasMoreHistory: bounded.hasMoreHistory,
56 latestLocalTurnOrdinal: bounded.latestLocalTurnOrdinal,
57 ...(compactProjection === null ? {} : { turnItemsOmitLocalVisible: true as const }),
58 + ...(input.compactIds ? { idTable: true as const } : {}),
59 };
60 }
61
@@ -62,6 +64,7 @@ export function buildBoundedThreadStreamSnapshot(input: {
64 readonly projection: OrchestrationV2ThreadProjection;
65 readonly compactTurnItems?: boolean;
66 readonly dedupeBodies?: boolean;
67 + readonly compactIds?: boolean;
68 }): ThreadSnapshotStreamItem {
69 const bounded = buildBoundedThreadProjection({
70 snapshotSequence: input.snapshotSequence,
@@ -74,6 +77,7 @@ export function buildBoundedThreadStreamSnapshot(input: {
77 bounded,
78 compactTurnItems: input.compactTurnItems === true,
79 dedupeBodies: input.dedupeBodies === true,
80 + compactIds: input.compactIds === true,
81 }),
82 };
83 }
apps/server/src/orchestration-v2/boundedSnapshotTransport.test.ts
+57
@@ -15,6 +15,7 @@ import {
15 ORCHESTRATION_PROTOCOL_HEADER,
16 ORCHESTRATION_PROTOCOL_VERSION,
17 OrchestrationV2ThreadBoundedSnapshot,
18 + OrchestrationV2ThreadHistoryPage,
19 OrchestrationV2ThreadStreamItem,
20 ProjectId,
21 ProviderDriverKind,
@@ -53,6 +54,9 @@ import * as ThreadManagementService from "./ThreadManagementService.ts";
54 const decodeBounded = Schema.decodeUnknownSync(
55 Schema.fromJsonString(Schema.toCodecJson(OrchestrationV2ThreadBoundedSnapshot)),
56 );
57 +const decodeHistoryPage = Schema.decodeUnknownSync(
58 + Schema.fromJsonString(Schema.toCodecJson(OrchestrationV2ThreadHistoryPage)),
59 +);
60 const decodeStreamItem = Schema.decodeUnknownSync(
61 Schema.toCodecJson(OrchestrationV2ThreadStreamItem),
62 );
@@ -81,6 +85,10 @@ const management = Layer.unwrap(
85 getThreadSnapshot: (id) => projections.getThreadSnapshot(id).pipe(Effect.orDie),
86 getThreadSnapshotWindow: (id, options) =>
87 projections.getThreadSnapshotWindow(id, options).pipe(Effect.orDie),
88 + getThreadHistoryPage: (id, cursor, throughEntryId, conversationOnly) =>
89 + projections
90 + .getThreadHistoryPage(id, cursor, throughEntryId, conversationOnly)
91 + .pipe(Effect.orDie),
92 streamStoredEventsFrom: (input) =>
93 input?.threadId === BODIES ? Stream.fromIterable(bodiesLiveTail()) : Stream.empty,
94 });
@@ -492,6 +500,46 @@ it.layer(TestLayer)("compact bounded snapshot transport", (it) => {
500 ),
501 );
502
503 + it.effect("HTTP: the ID table is opt-in and decodes to the same snapshot", () =>
504 + withHttp(async (get) => {
505 + const base = `/api/orchestration/threads/${encodeURIComponent(threadId)}/bounded?compactTurnItems=1`;
506 + const plainBody = await (await get(base)).text();
507 + const tableBody = await (await get(`${base}&compactIds=1`)).text();
508 + const wire = JSON.parse(tableBody);
509 +
510 + assert.notProperty(JSON.parse(plainBody), "ids");
511 + assert.isArray(wire.ids);
512 + assert.isNumber(wire.projection.visibleTurnItems[0].item.id);
513 + assert.isNumber(wire.projection.thread.id);
514 + assert.isBelow(tableBody.length, plainBody.length);
515 + const decoded = decodeBounded(tableBody);
516 + assert.deepStrictEqual(decoded, decodeBounded(plainBody));
517 +
518 + // Only the long thread has older history to page.
519 + if (threadId !== LONG) return;
520 + assert.isNotNull(decoded.historyCursor);
521 + const history = `/api/orchestration/threads/${encodeURIComponent(threadId)}/history?cursor=${encodeURIComponent(decoded.historyCursor ?? "")}`;
522 + const plainPage = await (await get(history)).text();
523 + const tablePage = await (await get(`${history}&compactIds=1`)).text();
524 + assert.isArray(JSON.parse(tablePage).ids);
525 + const page = decodeHistoryPage(tablePage);
526 + assert.isAbove(page.items.length, 0);
527 + assert.deepStrictEqual(page, decodeHistoryPage(plainPage));
528 + }),
529 + );
530 +
531 + it.effect("WS: the ID table is opt-in and decodes to the same snapshot", () =>
532 + Effect.gen(function* () {
533 + const input = { threadId, acceptBoundedSnapshot: true, acceptCompactTurnItems: true };
534 + const plain = yield* firstSnapshot(input);
535 + const table = yield* firstSnapshot({ ...input, acceptCompactIds: true });
536 +
537 + assert.notProperty(encodeStreamItem(plain.raw), "ids");
538 + assert.property(encodeStreamItem(table.raw), "ids");
539 + assert.deepStrictEqual(table.decoded, plain.decoded);
540 + }),
541 + );
542 +
543 it.effect("WS: bounded snapshots stay unchanged unless the client opts in", () =>
544 Effect.gen(function* () {
545 const legacy = yield* firstSnapshot({ threadId, acceptBoundedSnapshot: true });
@@ -546,6 +594,15 @@ it.layer(TestLayer)("compact bounded snapshot transport", (it) => {
594 boundedSnapshotProjection(decodeBounded(dedupedBody)),
595 boundedSnapshotProjection(decodeBounded(compactBody)),
596 );
597 + // The client sends both opt-ins: bodies restore after the ID table resolves.
598 + const tabledBody = await (
599 + await get(`${base}?compactTurnItems=1&dedupeBodies=1&compactIds=1`)
600 + ).text();
601 + assert.property(JSON.parse(tabledBody), "ids");
602 + assert.deepStrictEqual(
603 + boundedSnapshotProjection(decodeBounded(tabledBody)),
604 + boundedSnapshotProjection(decodeBounded(compactBody)),
605 + );
606 }),
607 );
608
apps/server/src/orchestration-v2/http.ts
+4
@@ -228,6 +228,7 @@ export const layer = HttpApiBuilder.group(
228 bounded,
229 compactTurnItems: args.query.compactTurnItems === "1",
230 dedupeBodies: args.query.dedupeBodies === "1",
231 + compactIds: args.query.compactIds === "1",
232 }),
233 };
234 }),
@@ -252,6 +253,9 @@ export const layer = HttpApiBuilder.group(
253 args.query.view === "conversation",
254 )
255 .pipe(
256 + Effect.map((page) =>
257 + args.query.compactIds === "1" ? { ...page, idTable: true as const } : page,
258 + ),
259 traceLocalHandlerWork,
260 Effect.catch(
261 Effect.fnUntraced(function* (cause) {
apps/server/src/ws.ts
+2
@@ -787,6 +787,7 @@ export const subscribeOrchestrationV2Thread = Effect.fn("ws.orchestrationV2.subs
787 readonly acceptBoundedSnapshot?: boolean;
788 readonly acceptCompactTurnItems?: boolean;
789 readonly acceptDedupedBodies?: boolean;
790 + readonly acceptCompactIds?: boolean;
791 }) {
792 const threadManagement = yield* ThreadManagementService.ThreadManagementService;
793 const dedupeBodies = input.acceptDedupedBodies === true;
@@ -890,6 +891,7 @@ export const subscribeOrchestrationV2Thread = Effect.fn("ws.orchestrationV2.subs
891 ...snapshot,
892 compactTurnItems: input.acceptCompactTurnItems === true,
893 dedupeBodies,
894 + compactIds: input.acceptCompactIds === true,
895 })
896 : (() => {
897 const projection = projectThreadProjectionForWire(snapshot.projection);
packages/client-runtime/src/state/boundedThreadSnapshotHttp.test.ts
+13 -5
@@ -1,7 +1,8 @@
1 -import { ThreadId } from "@t3tools/contracts";
1 +import { OrchestrationV2ThreadBoundedSnapshot, ThreadId } from "@t3tools/contracts";
2 import { describe, expect, it } from "@effect/vitest";
3 import * as Effect from "effect/Effect";
4 import * as Layer from "effect/Layer";
5 +import * as Schema from "effect/Schema";
6
7 import { PrimaryConnectionTarget, type PreparedConnection } from "../connection/model.ts";
8 import * as RpcHttp from "../rpc/http.ts";
@@ -117,22 +118,29 @@ const boundedBody = (compact: boolean) => ({
118 latestLocalTurnOrdinal: 2,
119 ...(compact ? { turnItemsOmitLocalVisible: true } : {}),
120 });
121 +const BoundedJson = Schema.toCodecJson(OrchestrationV2ThreadBoundedSnapshot);
122 +const idTableBody = () => {
123 + const snapshot = Schema.decodeUnknownSync(BoundedJson)(boundedBody(true));
124 + return Schema.encodeSync(BoundedJson)({ ...snapshot, idTable: true });
125 +};
126
127 describe("boundedThreadSnapshotLoader", () => {
128 it.effect.each([
123 - ["an older server that ignores the opt-in", false],
124 - ["a server that sends compact turnItems", true],
125 - ] as const)("requests compact turnItems and loads %s", ([, compact]) => {
129 + ["an older server that ignores the opt-in", () => boundedBody(false)],
130 + ["a server that sends compact turnItems", () => boundedBody(true)],
131 + ["a server that also sends an ID table", idTableBody],
132 + ] as const)("requests compact turnItems and loads %s", ([, body]) => {
133 const urls: string[] = [];
134 const fetchFn = ((input: RequestInfo | URL) => {
135 urls.push(String(input));
129 - return Promise.resolve(Response.json(boundedBody(compact)));
136 + return Promise.resolve(Response.json(body()));
137 }) satisfies typeof fetch;
138
139 return Effect.gen(function* () {
140 const loader = yield* ThreadSnapshotLoader.ThreadSnapshotLoader;
141 const result = yield* loader.load(PREPARED, THREAD_ID);
142 expect(new URL(urls[0]!).searchParams.get("compactTurnItems")).toBe("1");
143 + expect(new URL(urls[0]!).searchParams.get("compactIds")).toBe("1");
144 expect(result._tag).toBe("present");
145 if (result._tag !== "present") return;
146 expect(result.snapshot.projection.turnItems.map((item) => String(item.id))).toEqual([
packages/client-runtime/src/state/boundedThreadSnapshotHttp.ts
+1 -1
@@ -30,7 +30,7 @@ export const fetchEnvironmentBoundedThreadSnapshot = Effect.fn(
30 }) {
31 const endpoint = {
32 params: { threadId: input.threadId },
33 - query: { compactTurnItems: "1", dedupeBodies: "1" },
33 + query: { compactTurnItems: "1", dedupeBodies: "1", compactIds: "1" },
34 };
35 return yield* executeAuthenticatedEnvironmentHttpRequest({
36 ...input,
packages/client-runtime/src/state/threadHistoryHttp.ts
+1
@@ -28,6 +28,7 @@ export const fetchEnvironmentThreadHistoryPage = Effect.fn(
28 params: { threadId: input.threadId },
29 query: {
30 cursor: input.cursor,
31 + compactIds: "1",
32 ...(input.view === undefined ? {} : { view: input.view }),
33 ...(input.throughEntryId === undefined ? {} : { throughEntryId: input.throughEntryId }),
34 },
packages/client-runtime/src/state/threads.ts
+5 -1
@@ -961,7 +961,11 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make
961 ...(canResume ? { afterSequence: sequence } : {}),
962 ...(supportsCompletionMarker ? { requestCompletionMarker: true as const } : {}),
963 ...(acceptBoundedSnapshot
964 - ? { acceptBoundedSnapshot: true as const, acceptCompactTurnItems: true as const }
964 + ? {
965 + acceptBoundedSnapshot: true as const,
966 + acceptCompactTurnItems: true as const,
967 + acceptCompactIds: true as const,
968 + }
969 : {}),
970 acceptDedupedBodies: true as const,
971 };
packages/contracts/src/baseSchemas.ts
+9 -2
@@ -4,12 +4,19 @@ import * as Schema from "effect/Schema";
4 import * as SchemaGetter from "effect/SchemaGetter";
5 import * as SchemaTransformation from "effect/SchemaTransformation";
6
7 +// Hermes copies the string on every `trim()`, even when nothing changes.
8 +// Returning the original keeps repeated IDs shared after decode.
9 +const trim = (value: string) => {
10 + const trimmed = value.trim();
11 + return trimmed.length === value.length ? value : trimmed;
12 +};
13 +
14 export const TrimmedString = Schema.String.pipe(
15 Schema.decodeTo(
16 Schema.String,
17 SchemaTransformation.transformEffect({
11 - decode: (value) => Effect.succeed(value.trim()),
12 - encode: (value) => Effect.succeed(value.trim()),
18 + decode: (value) => Effect.succeed(trim(value)),
19 + encode: (value) => Effect.succeed(trim(value)),
20 }),
21 ),
22 );
packages/contracts/src/environmentHttp.ts
+2
@@ -674,12 +674,14 @@ const EnvironmentOrchestrationShellSnapshotQuery = Schema.Struct({
674 const EnvironmentOrchestrationThreadBoundedSnapshotQuery = Schema.Struct({
675 compactTurnItems: Schema.optionalKey(Schema.String),
676 dedupeBodies: Schema.optionalKey(Schema.String),
677 + compactIds: Schema.optionalKey(Schema.String),
678 });
679
680 const EnvironmentOrchestrationThreadHistoryQuery = Schema.Struct({
681 cursor: TrimmedNonEmptyString,
682 throughEntryId: Schema.optional(TrimmedNonEmptyString),
683 view: Schema.optional(Schema.Literals(["conversation", "activity"])),
684 + compactIds: Schema.optionalKey(Schema.String),
685 });
686
687 const EnvironmentOrchestrationThreadHistoryErrors = [
packages/contracts/src/orchestrationV2.test.ts
+18
@@ -175,6 +175,24 @@ describe("orchestration V2 contracts", () => {
175 if (snapshot?.kind !== "snapshot") throw new Error("expected a snapshot");
176 expect(snapshot.projection.turnItems.map((turnItem) => turnItem.id)).toEqual(["item-known"]);
177 expect(snapshot.projection.visibleTurnItems.map((row) => row.item.id)).toEqual(["item-known"]);
178 + // The same snapshot with an ID table: refs resolve before unknown items are skipped.
179 + const tabled = (turnItem: { readonly id: string }, ref: number) => ({ ...turnItem, id: ref });
180 + const [tabledSnapshot] = decodeWireItems([
181 + {
182 + kind: "snapshot",
183 + snapshotSequence: 1,
184 + projection: {
185 + ...projection,
186 + turnItems: [tabled(future, 0), tabled(known, 1)],
187 + visibleTurnItems: [
188 + { ...projected(0, future), sourceItemId: 0, item: tabled(future, 0) },
189 + { ...projected(1, known), sourceItemId: 1, item: tabled(known, 1) },
190 + ],
191 + },
192 + ids: ["item-future", "item-known"],
193 + },
194 + ]);
195 + expect(tabledSnapshot).toEqual(snapshot);
196 expect(futureEvent).toEqual({
197 kind: "unknown-event",
198 sequence: 2,
packages/contracts/src/orchestrationV2.ts
+65 -45
@@ -1,4 +1,5 @@
1 import { OrchestrationMessageContext } from "./composerContext.ts";
2 +import { withRecordIdTable } from "./orchestrationV2IdTable.ts";
3 import * as Effect from "effect/Effect";
4 import * as Schema from "effect/Schema";
5 import * as SchemaAST from "effect/SchemaAST";
@@ -3399,9 +3400,16 @@ export const OrchestrationV2SubscribeThreadInput = Schema.Struct({
3400 acceptCompactTurnItems: Schema.optionalKey(Schema.Boolean),
3401 /** The client restores `*Omitted` bodies from records it already holds. */
3402 acceptDedupedBodies: Schema.optionalKey(Schema.Boolean),
3403 + acceptCompactIds: Schema.optionalKey(Schema.Boolean),
3404 });
3405 export type OrchestrationV2SubscribeThreadInput = typeof OrchestrationV2SubscribeThreadInput.Type;
3406
3407 +/**
3408 + * Server-side only: set it on a response for a client that opted in, and the
3409 + * `withRecordIdTable` codec sends that response with an ID table.
3410 + */
3411 +const IdTableMarker = Schema.optionalKey(Schema.Literal(true));
3412 +
3413 export const OrchestrationV2ThreadDetailSnapshot = Schema.Struct({
3414 snapshotSequence: NonNegativeInt,
3415 projection: OrchestrationV2ThreadProjection,
@@ -3413,33 +3421,41 @@ export const OrchestrationV2ThreadDetailSnapshot = Schema.Struct({
3421 export type OrchestrationV2ThreadDetailSnapshot = typeof OrchestrationV2ThreadDetailSnapshot.Type;
3422
3423 /** Keep historyCursor opaque; resume events with afterSequence = snapshotSequence. */
3416 -export const OrchestrationV2ThreadBoundedSnapshot = Schema.Struct({
3417 - snapshotSequence: NonNegativeInt,
3418 - projection: OrchestrationV2ThreadProjection,
3419 - historyCursor: Schema.NullOr(TrimmedNonEmptyString),
3420 - hasMoreHistory: Schema.Boolean,
3421 - latestLocalTurnOrdinal: Schema.NullOr(NonNegativeInt),
3422 - /**
3423 - * Set only for clients that opted in: `projection.turnItems` omits the items
3424 - * of local visible rows, which lead the full list. Clients must restore them
3425 - * with `boundedSnapshotProjection` before using the projection.
3426 - */
3427 - turnItemsOmitLocalVisible: Schema.optionalKey(Schema.Literal(true)),
3428 -});
3424 +export const OrchestrationV2ThreadBoundedSnapshot = withRecordIdTable(
3425 + Schema.Struct({
3426 + snapshotSequence: NonNegativeInt,
3427 + projection: OrchestrationV2ThreadProjection,
3428 + historyCursor: Schema.NullOr(TrimmedNonEmptyString),
3429 + hasMoreHistory: Schema.Boolean,
3430 + latestLocalTurnOrdinal: Schema.NullOr(NonNegativeInt),
3431 + /**
3432 + * Set only for clients that opted in: `projection.turnItems` omits the items
3433 + * of local visible rows, which lead the full list. Clients must restore them
3434 + * with `boundedSnapshotProjection` before using the projection.
3435 + */
3436 + turnItemsOmitLocalVisible: Schema.optionalKey(Schema.Literal(true)),
3437 + idTable: IdTableMarker,
3438 + }),
3439 + () => "projection",
3440 +);
3441 export type OrchestrationV2ThreadBoundedSnapshot = typeof OrchestrationV2ThreadBoundedSnapshot.Type;
3442
3431 -export const OrchestrationV2ThreadHistoryPage = Schema.Struct({
3432 - snapshotSequence: NonNegativeInt,
3433 - items: projectedTurnItemArray(
3434 - OrchestrationV2ProjectedTurnItem,
3435 - OrchestrationV2ProjectedTurnItem.mapFields((fields) => ({
3436 - ...fields,
3437 - item: ForwardCompatibleUnion(OrchestrationV2TurnItem.members, "type"),
3438 - })),
3439 - ),
3440 - nextCursor: Schema.NullOr(TrimmedNonEmptyString),
3441 - hasMoreHistory: Schema.Boolean,
3442 -});
3443 +export const OrchestrationV2ThreadHistoryPage = withRecordIdTable(
3444 + Schema.Struct({
3445 + snapshotSequence: NonNegativeInt,
3446 + items: projectedTurnItemArray(
3447 + OrchestrationV2ProjectedTurnItem,
3448 + OrchestrationV2ProjectedTurnItem.mapFields((fields) => ({
3449 + ...fields,
3450 + item: ForwardCompatibleUnion(OrchestrationV2TurnItem.members, "type"),
3451 + })),
3452 + ),
3453 + nextCursor: Schema.NullOr(TrimmedNonEmptyString),
3454 + hasMoreHistory: Schema.Boolean,
3455 + idTable: IdTableMarker,
3456 + }),
3457 + () => null,
3458 +);
3459 export type OrchestrationV2ThreadHistoryPage = typeof OrchestrationV2ThreadHistoryPage.Type;
3460
3461 const knownDomainEventTypes: ReadonlySet<string> = new Set(
@@ -3486,26 +3502,30 @@ const OrchestrationV2UnknownThreadStreamEvent = Schema.Struct({
3502 ),
3503 );
3504
3489 -export const OrchestrationV2ThreadStreamItem = Schema.Union([
3490 - Schema.Struct({
3491 - kind: Schema.Literal("synchronized"),
3492 - }),
3493 - Schema.Struct({
3494 - kind: Schema.Literal("snapshot"),
3495 - snapshotSequence: NonNegativeInt,
3496 - projection: OrchestrationV2ThreadProjection,
3497 - historyCursor: Schema.optionalKey(Schema.NullOr(TrimmedNonEmptyString)),
3498 - hasMoreHistory: Schema.optionalKey(Schema.Boolean),
3499 - latestLocalTurnOrdinal: Schema.optionalKey(Schema.NullOr(NonNegativeInt)),
3500 - turnItemsOmitLocalVisible: Schema.optionalKey(Schema.Literal(true)),
3501 - }),
3502 - Schema.Struct({
3503 - kind: Schema.Literal("event"),
3504 - sequence: NonNegativeInt,
3505 - event: OrchestrationV2DomainEvent,
3506 - }),
3507 - OrchestrationV2UnknownThreadStreamEvent,
3508 -]);
3505 +export const OrchestrationV2ThreadStreamItem = withRecordIdTable(
3506 + Schema.Union([
3507 + Schema.Struct({
3508 + kind: Schema.Literal("synchronized"),
3509 + }),
3510 + Schema.Struct({
3511 + kind: Schema.Literal("snapshot"),
3512 + snapshotSequence: NonNegativeInt,
3513 + projection: OrchestrationV2ThreadProjection,
3514 + historyCursor: Schema.optionalKey(Schema.NullOr(TrimmedNonEmptyString)),
3515 + hasMoreHistory: Schema.optionalKey(Schema.Boolean),
3516 + latestLocalTurnOrdinal: Schema.optionalKey(Schema.NullOr(NonNegativeInt)),
3517 + turnItemsOmitLocalVisible: Schema.optionalKey(Schema.Literal(true)),
3518 + idTable: IdTableMarker,
3519 + }),
3520 + Schema.Struct({
3521 + kind: Schema.Literal("event"),
3522 + sequence: NonNegativeInt,
3523 + event: OrchestrationV2DomainEvent,
3524 + }),
3525 + OrchestrationV2UnknownThreadStreamEvent,
3526 + ]),
3527 + (item) => (item.kind === "snapshot" ? "projection" : undefined),
3528 +);
3529 export type OrchestrationV2ThreadStreamItem = typeof OrchestrationV2ThreadStreamItem.Type;
3530
3531 export class OrchestrationV2DispatchCommandError extends Schema.TaggedError<OrchestrationV2DispatchCommandError>()(
packages/contracts/src/orchestrationV2IdTable.test.ts new
+63
@@ -0,0 +1,63 @@
1 +import { describe, expect, it } from "@effect/vitest";
2 +import * as Schema from "effect/Schema";
3 +
4 +import { withRecordIdTable } from "./orchestrationV2IdTable.ts";
5 +
6 +const Page = withRecordIdTable(
7 + Schema.Struct({
8 + rows: Schema.Array(
9 + Schema.Struct({
10 + id: Schema.String,
11 + count: Schema.Number,
12 + parentIds: Schema.Array(Schema.String),
13 + mergedId: Schema.optional(Schema.String),
14 + item: Schema.Struct({ ownerId: Schema.NullOr(Schema.String), legacyId: Schema.Number }),
15 + }),
16 + ),
17 + idTable: Schema.optionalKey(Schema.Literal(true)),
18 + }),
19 + () => null,
20 +);
21 +const encode = Schema.encodeSync(Schema.toCodecJson(Page));
22 +const decode = Schema.decodeUnknownSync(Schema.toCodecJson(Page));
23 +
24 +describe("record ID table", () => {
25 + it("interns record ID slots and restores them exactly, numbers included", () => {
26 + const value = {
27 + rows: [
28 + { id: "row:a", count: 7, parentIds: ["row:b"], item: { ownerId: "row:b", legacyId: 3 } },
29 + { id: "row:b", count: 3, parentIds: [], item: { ownerId: null, legacyId: 0 } },
30 + ],
31 + };
32 + const wire = JSON.parse(JSON.stringify(encode({ ...value, idTable: true })));
33 +
34 + expect(wire).toEqual({
35 + rows: [
36 + { id: 0, count: 7, parentIds: [1], item: { ownerId: 1, legacyId: 2 } },
37 + { id: 1, count: 3, parentIds: [], item: { ownerId: null, legacyId: 3 } },
38 + ],
39 + ids: ["row:a", "row:b", 3, 0],
40 + });
41 + expect(decode(wire)).toEqual(value);
42 + // The table is consumed, so a second decode of the same object cannot remap numbers.
43 + expect(decode(wire)).toEqual(value);
44 + });
45 +
46 + it("rejects an index outside the table, even in an optional slot", () => {
47 + const wire = {
48 + rows: [{ id: 0, count: 1, parentIds: [], mergedId: 5, item: { ownerId: null, legacyId: 0 } }],
49 + ids: ["row:a"],
50 + };
51 +
52 + expect(() => decode(wire)).toThrow(/mergedId/);
53 + });
54 +
55 + it("sends and reads the plain form without the marker", () => {
56 + const value = {
57 + rows: [{ id: "row:a", count: 1, parentIds: [], item: { ownerId: null, legacyId: 1 } }],
58 + };
59 +
60 + expect(encode(value)).toEqual(value);
61 + expect(decode(value)).toEqual(value);
62 + });
63 +});
packages/contracts/src/orchestrationV2IdTable.ts new
+125
@@ -0,0 +1,125 @@
1 +import * as Schema from "effect/Schema";
2 +import * as SchemaGetter from "effect/SchemaGetter";
3 +
4 +/**
5 + * Opt-in wire form: on each record (top-level array element, top-level object, or `item`), values
6 + * under `id`, `*Id`, and `*Ids` index the response's `ids` table. Both peers must apply this exact
7 + * rule; changing it needs a new opt-in (`compactIds=2`, or a new `accept*` key over WS).
8 + */
9 +type JsonObject = { [key: string]: unknown };
10 +
11 +const isObject = (value: unknown): value is JsonObject =>
12 + typeof value === "object" && value !== null && !Array.isArray(value);
13 +const isIdKey = (key: string) => key === "id" || key.endsWith("Id");
14 +const isIdListKey = (key: string) => key.endsWith("Ids");
15 +
16 +function internRecordIds(container: JsonObject, ids: Array<string | number>): JsonObject {
17 + const index = new Map<string | number, number>();
18 + const ref = (value: unknown) => {
19 + if (typeof value !== "string" && typeof value !== "number") return value;
20 + let position = index.get(value);
21 + if (position === undefined) {
22 + position = ids.length;
23 + ids.push(value);
24 + index.set(value, position);
25 + }
26 + return position;
27 + };
28 + // Copies each record: unknown members can pass through encoding by reference.
29 + const internRecord = (record: JsonObject): JsonObject => {
30 + const out: JsonObject = {};
31 + for (const key in record) {
32 + const value = record[key];
33 + out[key] = isIdKey(key)
34 + ? ref(value)
35 + : isIdListKey(key) && Array.isArray(value)
36 + ? value.map(ref)
37 + : key === "item" && isObject(value)
38 + ? internRecord(value)
39 + : value;
40 + }
41 + return out;
42 + };
43 + const out: JsonObject = {};
44 + for (const key in container) {
45 + const value = container[key];
46 + out[key] = Array.isArray(value)
47 + ? value.map((record) => (isObject(record) ? internRecord(record) : record))
48 + : isObject(value)
49 + ? internRecord(value)
50 + : value;
51 + }
52 + return out;
53 +}
54 +
55 +// Mutates freshly parsed JSON in place; the client owns it and this avoids a copy.
56 +function expandRecordIds(container: JsonObject, ids: ReadonlyArray<unknown>): void {
57 + const expandRecord = (record: JsonObject) => {
58 + for (const key in record) {
59 + const value = record[key];
60 + if (typeof value === "number") {
61 + if (isIdKey(key)) record[key] = ids[value];
62 + } else if (Array.isArray(value)) {
63 + if (isIdListKey(key)) {
64 + for (let i = 0; i < value.length; i++) {
65 + const entry = value[i];
66 + if (typeof entry === "number") value[i] = ids[entry];
67 + }
68 + }
69 + } else if (key === "item" && isObject(value)) {
70 + expandRecord(value);
71 + }
72 + }
73 + };
74 + for (const key in container) {
75 + const value = container[key];
76 + if (Array.isArray(value)) {
77 + for (const record of value) if (isObject(record)) expandRecord(record);
78 + } else if (isObject(value)) {
79 + expandRecord(value);
80 + }
81 + }
82 +}
83 +
84 +// `toCodecJson` turns `Schema.Unknown` into a recursive JSON check of the whole
85 +// tree. The input is parsed JSON and the inner codec validates it anyway.
86 +const ParsedJson = Schema.declare((_: unknown): _ is unknown => true, {
87 + toCodecJson: () => undefined,
88 +});
89 +
90 +/**
91 + * Adds the optional ID table to a snapshot-shaped schema. `container` returns the
92 + * key whose value holds the records, `null` when records sit at the top level, or
93 + * `undefined` when the value has no records (other stream item kinds).
94 + */
95 +export const withRecordIdTable = <S extends Schema.Top>(
96 + schema: S,
97 + container: (value: JsonObject) => string | null | undefined,
98 +) => {
99 + const codec = Schema.toCodecJson(schema);
100 + type Encoded = typeof codec.Encoded;
101 + return ParsedJson.pipe(
102 + Schema.decodeTo(codec, {
103 + decode: SchemaGetter.transform((raw: unknown): Encoded => {
104 + if (!isObject(raw) || !Array.isArray(raw.ids)) return raw as Encoded;
105 + // Consumed on the parsed object itself, so decoding it again is a no-op.
106 + const ids: ReadonlyArray<unknown> = raw.ids;
107 + delete raw.ids;
108 + const key = container(raw);
109 + const target = key === null ? raw : key === undefined ? undefined : raw[key];
110 + if (isObject(target)) expandRecordIds(target, ids);
111 + return raw as Encoded;
112 + }),
113 + encode: SchemaGetter.transform((encoded: Encoded): unknown => {
114 + if (!isObject(encoded) || encoded.idTable !== true) return encoded;
115 + const { idTable: _idTable, ...rest } = encoded;
116 + const key = container(rest);
117 + const ids: Array<string | number> = [];
118 + if (key === null) return { ...internRecordIds(rest, ids), ids };
119 + const target = key === undefined ? undefined : rest[key];
120 + if (key === undefined || !isObject(target)) return rest;
121 + return { ...rest, [key]: internRecordIds(target, ids), ids };
122 + }),
123 + }),
124 + );
125 +};