The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 18:15 UTC
perf(orchestration): thread snapshots send each long ID once (#24643)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Theo Browne committed
Oct 24, 2026 at 20:02 UTC
80455376f01f19c10a06dc08b0629486c017bf79
14 files changed
+369
-54
apps/server/src/orchestration-v2/ThreadStream.ts
+4
@@ -41,6 +41,7 @@ export function boundedSnapshotResponseFields(input: {
41
readonly bounded: BoundedProjectionResult;
42
readonly compactTurnItems: boolean;
43
readonly dedupeBodies: boolean;
44
+ readonly compactIds: boolean;
45
}) {
46
const { bounded } = input;
47
// Dedupe keeps shared items one object, which the compaction's identity check needs.
@@ -54,6 +55,7 @@ export function boundedSnapshotResponseFields(input: {
55
hasMoreHistory: bounded.hasMoreHistory,
56
latestLocalTurnOrdinal: bounded.latestLocalTurnOrdinal,
57
...(compactProjection === null ? {} : { turnItemsOmitLocalVisible: true as const }),
58
+ ...(input.compactIds ? { idTable: true as const } : {}),
59
};
60
}
61
@@ -62,6 +64,7 @@ export function buildBoundedThreadStreamSnapshot(input: {
64
readonly projection: OrchestrationV2ThreadProjection;
65
readonly compactTurnItems?: boolean;
66
readonly dedupeBodies?: boolean;
67
+ readonly compactIds?: boolean;
68
}): ThreadSnapshotStreamItem {
69
const bounded = buildBoundedThreadProjection({
70
snapshotSequence: input.snapshotSequence,
@@ -74,6 +77,7 @@ export function buildBoundedThreadStreamSnapshot(input: {
77
bounded,
78
compactTurnItems: input.compactTurnItems === true,
79
dedupeBodies: input.dedupeBodies === true,
80
+ compactIds: input.compactIds === true,
81
}),
82
};
83
}
apps/server/src/orchestration-v2/boundedSnapshotTransport.test.ts
+57
@@ -15,6 +15,7 @@ import {
15
ORCHESTRATION_PROTOCOL_HEADER,
16
ORCHESTRATION_PROTOCOL_VERSION,
17
OrchestrationV2ThreadBoundedSnapshot,
18
+ OrchestrationV2ThreadHistoryPage,
19
OrchestrationV2ThreadStreamItem,
20
ProjectId,
21
ProviderDriverKind,
@@ -53,6 +54,9 @@ import * as ThreadManagementService from "./ThreadManagementService.ts";
54
const decodeBounded = Schema.decodeUnknownSync(
55
Schema.fromJsonString(Schema.toCodecJson(OrchestrationV2ThreadBoundedSnapshot)),
56
);
57
+const decodeHistoryPage = Schema.decodeUnknownSync(
58
+ Schema.fromJsonString(Schema.toCodecJson(OrchestrationV2ThreadHistoryPage)),
59
+);
60
const decodeStreamItem = Schema.decodeUnknownSync(
61
Schema.toCodecJson(OrchestrationV2ThreadStreamItem),
62
);
@@ -81,6 +85,10 @@ const management = Layer.unwrap(
85
getThreadSnapshot: (id) => projections.getThreadSnapshot(id).pipe(Effect.orDie),
86
getThreadSnapshotWindow: (id, options) =>
87
projections.getThreadSnapshotWindow(id, options).pipe(Effect.orDie),
88
+ getThreadHistoryPage: (id, cursor, throughEntryId, conversationOnly) =>
89
+ projections
90
+ .getThreadHistoryPage(id, cursor, throughEntryId, conversationOnly)
91
+ .pipe(Effect.orDie),
92
streamStoredEventsFrom: (input) =>
93
input?.threadId === BODIES ? Stream.fromIterable(bodiesLiveTail()) : Stream.empty,
94
});
@@ -492,6 +500,46 @@ it.layer(TestLayer)("compact bounded snapshot transport", (it) => {
500
),
501
);
502
503
+ it.effect("HTTP: the ID table is opt-in and decodes to the same snapshot", () =>
504
+ withHttp(async (get) => {
505
+ const base = `/api/orchestration/threads/${encodeURIComponent(threadId)}/bounded?compactTurnItems=1`;
506
+ const plainBody = await (await get(base)).text();
507
+ const tableBody = await (await get(`${base}&compactIds=1`)).text();
508
+ const wire = JSON.parse(tableBody);
509
+
510
+ assert.notProperty(JSON.parse(plainBody), "ids");
511
+ assert.isArray(wire.ids);
512
+ assert.isNumber(wire.projection.visibleTurnItems[0].item.id);
513
+ assert.isNumber(wire.projection.thread.id);
514
+ assert.isBelow(tableBody.length, plainBody.length);
515
+ const decoded = decodeBounded(tableBody);
516
+ assert.deepStrictEqual(decoded, decodeBounded(plainBody));
517
+
518
+ // Only the long thread has older history to page.
519
+ if (threadId !== LONG) return;
520
+ assert.isNotNull(decoded.historyCursor);
521
+ const history = `/api/orchestration/threads/${encodeURIComponent(threadId)}/history?cursor=${encodeURIComponent(decoded.historyCursor ?? "")}`;
522
+ const plainPage = await (await get(history)).text();
523
+ const tablePage = await (await get(`${history}&compactIds=1`)).text();
524
+ assert.isArray(JSON.parse(tablePage).ids);
525
+ const page = decodeHistoryPage(tablePage);
526
+ assert.isAbove(page.items.length, 0);
527
+ assert.deepStrictEqual(page, decodeHistoryPage(plainPage));
528
+ }),
529
+ );
530
+
531
+ it.effect("WS: the ID table is opt-in and decodes to the same snapshot", () =>
532
+ Effect.gen(function* () {
533
+ const input = { threadId, acceptBoundedSnapshot: true, acceptCompactTurnItems: true };
534
+ const plain = yield* firstSnapshot(input);
535
+ const table = yield* firstSnapshot({ ...input, acceptCompactIds: true });
536
+
537
+ assert.notProperty(encodeStreamItem(plain.raw), "ids");
538
+ assert.property(encodeStreamItem(table.raw), "ids");
539
+ assert.deepStrictEqual(table.decoded, plain.decoded);
540
+ }),
541
+ );
542
+
543
it.effect("WS: bounded snapshots stay unchanged unless the client opts in", () =>
544
Effect.gen(function* () {
545
const legacy = yield* firstSnapshot({ threadId, acceptBoundedSnapshot: true });
@@ -546,6 +594,15 @@ it.layer(TestLayer)("compact bounded snapshot transport", (it) => {
594
boundedSnapshotProjection(decodeBounded(dedupedBody)),
595
boundedSnapshotProjection(decodeBounded(compactBody)),
596
);
597
+ // The client sends both opt-ins: bodies restore after the ID table resolves.
598
+ const tabledBody = await (
599
+ await get(`${base}?compactTurnItems=1&dedupeBodies=1&compactIds=1`)
600
+ ).text();
601
+ assert.property(JSON.parse(tabledBody), "ids");
602
+ assert.deepStrictEqual(
603
+ boundedSnapshotProjection(decodeBounded(tabledBody)),
604
+ boundedSnapshotProjection(decodeBounded(compactBody)),
605
+ );
606
}),
607
);
608
apps/server/src/orchestration-v2/http.ts
+4
@@ -228,6 +228,7 @@ export const layer = HttpApiBuilder.group(
228
bounded,
229
compactTurnItems: args.query.compactTurnItems === "1",
230
dedupeBodies: args.query.dedupeBodies === "1",
231
+ compactIds: args.query.compactIds === "1",
232
}),
233
};
234
}),
@@ -252,6 +253,9 @@ export const layer = HttpApiBuilder.group(
253
args.query.view === "conversation",
254
)
255
.pipe(
256
+ Effect.map((page) =>
257
+ args.query.compactIds === "1" ? { ...page, idTable: true as const } : page,
258
+ ),
259
traceLocalHandlerWork,
260
Effect.catch(
261
Effect.fnUntraced(function* (cause) {
apps/server/src/ws.ts
+2
@@ -787,6 +787,7 @@ export const subscribeOrchestrationV2Thread = Effect.fn("ws.orchestrationV2.subs
787
readonly acceptBoundedSnapshot?: boolean;
788
readonly acceptCompactTurnItems?: boolean;
789
readonly acceptDedupedBodies?: boolean;
790
+ readonly acceptCompactIds?: boolean;
791
}) {
792
const threadManagement = yield* ThreadManagementService.ThreadManagementService;
793
const dedupeBodies = input.acceptDedupedBodies === true;
@@ -890,6 +891,7 @@ export const subscribeOrchestrationV2Thread = Effect.fn("ws.orchestrationV2.subs
891
...snapshot,
892
compactTurnItems: input.acceptCompactTurnItems === true,
893
dedupeBodies,
894
+ compactIds: input.acceptCompactIds === true,
895
})
896
: (() => {
897
const projection = projectThreadProjectionForWire(snapshot.projection);
packages/client-runtime/src/state/boundedThreadSnapshotHttp.test.ts
+13
-5
@@ -1,7 +1,8 @@
1
-import { ThreadId } from "@t3tools/contracts";
1
+import { OrchestrationV2ThreadBoundedSnapshot, ThreadId } from "@t3tools/contracts";
2
import { describe, expect, it } from "@effect/vitest";
3
import * as Effect from "effect/Effect";
4
import * as Layer from "effect/Layer";
5
+import * as Schema from "effect/Schema";
6
7
import { PrimaryConnectionTarget, type PreparedConnection } from "../connection/model.ts";
8
import * as RpcHttp from "../rpc/http.ts";
@@ -117,22 +118,29 @@ const boundedBody = (compact: boolean) => ({
118
latestLocalTurnOrdinal: 2,
119
...(compact ? { turnItemsOmitLocalVisible: true } : {}),
120
});
121
+const BoundedJson = Schema.toCodecJson(OrchestrationV2ThreadBoundedSnapshot);
122
+const idTableBody = () => {
123
+ const snapshot = Schema.decodeUnknownSync(BoundedJson)(boundedBody(true));
124
+ return Schema.encodeSync(BoundedJson)({ ...snapshot, idTable: true });
125
+};
126
127
describe("boundedThreadSnapshotLoader", () => {
128
it.effect.each([
123
- ["an older server that ignores the opt-in", false],
124
- ["a server that sends compact turnItems", true],
125
- ] as const)("requests compact turnItems and loads %s", ([, compact]) => {
129
+ ["an older server that ignores the opt-in", () => boundedBody(false)],
130
+ ["a server that sends compact turnItems", () => boundedBody(true)],
131
+ ["a server that also sends an ID table", idTableBody],
132
+ ] as const)("requests compact turnItems and loads %s", ([, body]) => {
133
const urls: string[] = [];
134
const fetchFn = ((input: RequestInfo | URL) => {
135
urls.push(String(input));
129
- return Promise.resolve(Response.json(boundedBody(compact)));
136
+ return Promise.resolve(Response.json(body()));
137
}) satisfies typeof fetch;
138
139
return Effect.gen(function* () {
140
const loader = yield* ThreadSnapshotLoader.ThreadSnapshotLoader;
141
const result = yield* loader.load(PREPARED, THREAD_ID);
142
expect(new URL(urls[0]!).searchParams.get("compactTurnItems")).toBe("1");
143
+ expect(new URL(urls[0]!).searchParams.get("compactIds")).toBe("1");
144
expect(result._tag).toBe("present");
145
if (result._tag !== "present") return;
146
expect(result.snapshot.projection.turnItems.map((item) => String(item.id))).toEqual([
packages/client-runtime/src/state/boundedThreadSnapshotHttp.ts
+1
-1
@@ -30,7 +30,7 @@ export const fetchEnvironmentBoundedThreadSnapshot = Effect.fn(
30
}) {
31
const endpoint = {
32
params: { threadId: input.threadId },
33
- query: { compactTurnItems: "1", dedupeBodies: "1" },
33
+ query: { compactTurnItems: "1", dedupeBodies: "1", compactIds: "1" },
34
};
35
return yield* executeAuthenticatedEnvironmentHttpRequest({
36
...input,
packages/client-runtime/src/state/threadHistoryHttp.ts
+1
@@ -28,6 +28,7 @@ export const fetchEnvironmentThreadHistoryPage = Effect.fn(
28
params: { threadId: input.threadId },
29
query: {
30
cursor: input.cursor,
31
+ compactIds: "1",
32
...(input.view === undefined ? {} : { view: input.view }),
33
...(input.throughEntryId === undefined ? {} : { throughEntryId: input.throughEntryId }),
34
},
packages/client-runtime/src/state/threads.ts
+5
-1
@@ -961,7 +961,11 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make
961
...(canResume ? { afterSequence: sequence } : {}),
962
...(supportsCompletionMarker ? { requestCompletionMarker: true as const } : {}),
963
...(acceptBoundedSnapshot
964
- ? { acceptBoundedSnapshot: true as const, acceptCompactTurnItems: true as const }
964
+ ? {
965
+ acceptBoundedSnapshot: true as const,
966
+ acceptCompactTurnItems: true as const,
967
+ acceptCompactIds: true as const,
968
+ }
969
: {}),
970
acceptDedupedBodies: true as const,
971
};
packages/contracts/src/baseSchemas.ts
+9
-2
@@ -4,12 +4,19 @@ import * as Schema from "effect/Schema";
4
import * as SchemaGetter from "effect/SchemaGetter";
5
import * as SchemaTransformation from "effect/SchemaTransformation";
6
7
+// Hermes copies the string on every `trim()`, even when nothing changes.
8
+// Returning the original keeps repeated IDs shared after decode.
9
+const trim = (value: string) => {
10
+ const trimmed = value.trim();
11
+ return trimmed.length === value.length ? value : trimmed;
12
+};
13
+
14
export const TrimmedString = Schema.String.pipe(
15
Schema.decodeTo(
16
Schema.String,
17
SchemaTransformation.transformEffect({
11
- decode: (value) => Effect.succeed(value.trim()),
12
- encode: (value) => Effect.succeed(value.trim()),
18
+ decode: (value) => Effect.succeed(trim(value)),
19
+ encode: (value) => Effect.succeed(trim(value)),
20
}),
21
),
22
);
packages/contracts/src/environmentHttp.ts
+2
@@ -674,12 +674,14 @@ const EnvironmentOrchestrationShellSnapshotQuery = Schema.Struct({
674
const EnvironmentOrchestrationThreadBoundedSnapshotQuery = Schema.Struct({
675
compactTurnItems: Schema.optionalKey(Schema.String),
676
dedupeBodies: Schema.optionalKey(Schema.String),
677
+ compactIds: Schema.optionalKey(Schema.String),
678
});
679
680
const EnvironmentOrchestrationThreadHistoryQuery = Schema.Struct({
681
cursor: TrimmedNonEmptyString,
682
throughEntryId: Schema.optional(TrimmedNonEmptyString),
683
view: Schema.optional(Schema.Literals(["conversation", "activity"])),
684
+ compactIds: Schema.optionalKey(Schema.String),
685
});
686
687
const EnvironmentOrchestrationThreadHistoryErrors = [
packages/contracts/src/orchestrationV2.test.ts
+18
@@ -175,6 +175,24 @@ describe("orchestration V2 contracts", () => {
175
if (snapshot?.kind !== "snapshot") throw new Error("expected a snapshot");
176
expect(snapshot.projection.turnItems.map((turnItem) => turnItem.id)).toEqual(["item-known"]);
177
expect(snapshot.projection.visibleTurnItems.map((row) => row.item.id)).toEqual(["item-known"]);
178
+ // The same snapshot with an ID table: refs resolve before unknown items are skipped.
179
+ const tabled = (turnItem: { readonly id: string }, ref: number) => ({ ...turnItem, id: ref });
180
+ const [tabledSnapshot] = decodeWireItems([
181
+ {
182
+ kind: "snapshot",
183
+ snapshotSequence: 1,
184
+ projection: {
185
+ ...projection,
186
+ turnItems: [tabled(future, 0), tabled(known, 1)],
187
+ visibleTurnItems: [
188
+ { ...projected(0, future), sourceItemId: 0, item: tabled(future, 0) },
189
+ { ...projected(1, known), sourceItemId: 1, item: tabled(known, 1) },
190
+ ],
191
+ },
192
+ ids: ["item-future", "item-known"],
193
+ },
194
+ ]);
195
+ expect(tabledSnapshot).toEqual(snapshot);
196
expect(futureEvent).toEqual({
197
kind: "unknown-event",
198
sequence: 2,
packages/contracts/src/orchestrationV2.ts
+65
-45
@@ -1,4 +1,5 @@
1
import { OrchestrationMessageContext } from "./composerContext.ts";
2
+import { withRecordIdTable } from "./orchestrationV2IdTable.ts";
3
import * as Effect from "effect/Effect";
4
import * as Schema from "effect/Schema";
5
import * as SchemaAST from "effect/SchemaAST";
@@ -3399,9 +3400,16 @@ export const OrchestrationV2SubscribeThreadInput = Schema.Struct({
3400
acceptCompactTurnItems: Schema.optionalKey(Schema.Boolean),
3401
/** The client restores `*Omitted` bodies from records it already holds. */
3402
acceptDedupedBodies: Schema.optionalKey(Schema.Boolean),
3403
+ acceptCompactIds: Schema.optionalKey(Schema.Boolean),
3404
});
3405
export type OrchestrationV2SubscribeThreadInput = typeof OrchestrationV2SubscribeThreadInput.Type;
3406
3407
+/**
3408
+ * Server-side only: set it on a response for a client that opted in, and the
3409
+ * `withRecordIdTable` codec sends that response with an ID table.
3410
+ */
3411
+const IdTableMarker = Schema.optionalKey(Schema.Literal(true));
3412
+
3413
export const OrchestrationV2ThreadDetailSnapshot = Schema.Struct({
3414
snapshotSequence: NonNegativeInt,
3415
projection: OrchestrationV2ThreadProjection,
@@ -3413,33 +3421,41 @@ export const OrchestrationV2ThreadDetailSnapshot = Schema.Struct({
3421
export type OrchestrationV2ThreadDetailSnapshot = typeof OrchestrationV2ThreadDetailSnapshot.Type;
3422
3423
/** Keep historyCursor opaque; resume events with afterSequence = snapshotSequence. */
3416
-export const OrchestrationV2ThreadBoundedSnapshot = Schema.Struct({
3417
- snapshotSequence: NonNegativeInt,
3418
- projection: OrchestrationV2ThreadProjection,
3419
- historyCursor: Schema.NullOr(TrimmedNonEmptyString),
3420
- hasMoreHistory: Schema.Boolean,
3421
- latestLocalTurnOrdinal: Schema.NullOr(NonNegativeInt),
3422
- /**
3423
- * Set only for clients that opted in: `projection.turnItems` omits the items
3424
- * of local visible rows, which lead the full list. Clients must restore them
3425
- * with `boundedSnapshotProjection` before using the projection.
3426
- */
3427
- turnItemsOmitLocalVisible: Schema.optionalKey(Schema.Literal(true)),
3428
-});
3424
+export const OrchestrationV2ThreadBoundedSnapshot = withRecordIdTable(
3425
+ Schema.Struct({
3426
+ snapshotSequence: NonNegativeInt,
3427
+ projection: OrchestrationV2ThreadProjection,
3428
+ historyCursor: Schema.NullOr(TrimmedNonEmptyString),
3429
+ hasMoreHistory: Schema.Boolean,
3430
+ latestLocalTurnOrdinal: Schema.NullOr(NonNegativeInt),
3431
+ /**
3432
+ * Set only for clients that opted in: `projection.turnItems` omits the items
3433
+ * of local visible rows, which lead the full list. Clients must restore them
3434
+ * with `boundedSnapshotProjection` before using the projection.
3435
+ */
3436
+ turnItemsOmitLocalVisible: Schema.optionalKey(Schema.Literal(true)),
3437
+ idTable: IdTableMarker,
3438
+ }),
3439
+ () => "projection",
3440
+);
3441
export type OrchestrationV2ThreadBoundedSnapshot = typeof OrchestrationV2ThreadBoundedSnapshot.Type;
3442
3431
-export const OrchestrationV2ThreadHistoryPage = Schema.Struct({
3432
- snapshotSequence: NonNegativeInt,
3433
- items: projectedTurnItemArray(
3434
- OrchestrationV2ProjectedTurnItem,
3435
- OrchestrationV2ProjectedTurnItem.mapFields((fields) => ({
3436
- ...fields,
3437
- item: ForwardCompatibleUnion(OrchestrationV2TurnItem.members, "type"),
3438
- })),
3439
- ),
3440
- nextCursor: Schema.NullOr(TrimmedNonEmptyString),
3441
- hasMoreHistory: Schema.Boolean,
3442
-});
3443
+export const OrchestrationV2ThreadHistoryPage = withRecordIdTable(
3444
+ Schema.Struct({
3445
+ snapshotSequence: NonNegativeInt,
3446
+ items: projectedTurnItemArray(
3447
+ OrchestrationV2ProjectedTurnItem,
3448
+ OrchestrationV2ProjectedTurnItem.mapFields((fields) => ({
3449
+ ...fields,
3450
+ item: ForwardCompatibleUnion(OrchestrationV2TurnItem.members, "type"),
3451
+ })),
3452
+ ),
3453
+ nextCursor: Schema.NullOr(TrimmedNonEmptyString),
3454
+ hasMoreHistory: Schema.Boolean,
3455
+ idTable: IdTableMarker,
3456
+ }),
3457
+ () => null,
3458
+);
3459
export type OrchestrationV2ThreadHistoryPage = typeof OrchestrationV2ThreadHistoryPage.Type;
3460
3461
const knownDomainEventTypes: ReadonlySet<string> = new Set(
@@ -3486,26 +3502,30 @@ const OrchestrationV2UnknownThreadStreamEvent = Schema.Struct({
3502
),
3503
);
3504
3489
-export const OrchestrationV2ThreadStreamItem = Schema.Union([
3490
- Schema.Struct({
3491
- kind: Schema.Literal("synchronized"),
3492
- }),
3493
- Schema.Struct({
3494
- kind: Schema.Literal("snapshot"),
3495
- snapshotSequence: NonNegativeInt,
3496
- projection: OrchestrationV2ThreadProjection,
3497
- historyCursor: Schema.optionalKey(Schema.NullOr(TrimmedNonEmptyString)),
3498
- hasMoreHistory: Schema.optionalKey(Schema.Boolean),
3499
- latestLocalTurnOrdinal: Schema.optionalKey(Schema.NullOr(NonNegativeInt)),
3500
- turnItemsOmitLocalVisible: Schema.optionalKey(Schema.Literal(true)),
3501
- }),
3502
- Schema.Struct({
3503
- kind: Schema.Literal("event"),
3504
- sequence: NonNegativeInt,
3505
- event: OrchestrationV2DomainEvent,
3506
- }),
3507
- OrchestrationV2UnknownThreadStreamEvent,
3508
-]);
3505
+export const OrchestrationV2ThreadStreamItem = withRecordIdTable(
3506
+ Schema.Union([
3507
+ Schema.Struct({
3508
+ kind: Schema.Literal("synchronized"),
3509
+ }),
3510
+ Schema.Struct({
3511
+ kind: Schema.Literal("snapshot"),
3512
+ snapshotSequence: NonNegativeInt,
3513
+ projection: OrchestrationV2ThreadProjection,
3514
+ historyCursor: Schema.optionalKey(Schema.NullOr(TrimmedNonEmptyString)),
3515
+ hasMoreHistory: Schema.optionalKey(Schema.Boolean),
3516
+ latestLocalTurnOrdinal: Schema.optionalKey(Schema.NullOr(NonNegativeInt)),
3517
+ turnItemsOmitLocalVisible: Schema.optionalKey(Schema.Literal(true)),
3518
+ idTable: IdTableMarker,
3519
+ }),
3520
+ Schema.Struct({
3521
+ kind: Schema.Literal("event"),
3522
+ sequence: NonNegativeInt,
3523
+ event: OrchestrationV2DomainEvent,
3524
+ }),
3525
+ OrchestrationV2UnknownThreadStreamEvent,
3526
+ ]),
3527
+ (item) => (item.kind === "snapshot" ? "projection" : undefined),
3528
+);
3529
export type OrchestrationV2ThreadStreamItem = typeof OrchestrationV2ThreadStreamItem.Type;
3530
3531
export class OrchestrationV2DispatchCommandError extends Schema.TaggedError<OrchestrationV2DispatchCommandError>()(
packages/contracts/src/orchestrationV2IdTable.test.ts
new
+63
@@ -0,0 +1,63 @@
1
+import { describe, expect, it } from "@effect/vitest";
2
+import * as Schema from "effect/Schema";
3
+
4
+import { withRecordIdTable } from "./orchestrationV2IdTable.ts";
5
+
6
+const Page = withRecordIdTable(
7
+ Schema.Struct({
8
+ rows: Schema.Array(
9
+ Schema.Struct({
10
+ id: Schema.String,
11
+ count: Schema.Number,
12
+ parentIds: Schema.Array(Schema.String),
13
+ mergedId: Schema.optional(Schema.String),
14
+ item: Schema.Struct({ ownerId: Schema.NullOr(Schema.String), legacyId: Schema.Number }),
15
+ }),
16
+ ),
17
+ idTable: Schema.optionalKey(Schema.Literal(true)),
18
+ }),
19
+ () => null,
20
+);
21
+const encode = Schema.encodeSync(Schema.toCodecJson(Page));
22
+const decode = Schema.decodeUnknownSync(Schema.toCodecJson(Page));
23
+
24
+describe("record ID table", () => {
25
+ it("interns record ID slots and restores them exactly, numbers included", () => {
26
+ const value = {
27
+ rows: [
28
+ { id: "row:a", count: 7, parentIds: ["row:b"], item: { ownerId: "row:b", legacyId: 3 } },
29
+ { id: "row:b", count: 3, parentIds: [], item: { ownerId: null, legacyId: 0 } },
30
+ ],
31
+ };
32
+ const wire = JSON.parse(JSON.stringify(encode({ ...value, idTable: true })));
33
+
34
+ expect(wire).toEqual({
35
+ rows: [
36
+ { id: 0, count: 7, parentIds: [1], item: { ownerId: 1, legacyId: 2 } },
37
+ { id: 1, count: 3, parentIds: [], item: { ownerId: null, legacyId: 3 } },
38
+ ],
39
+ ids: ["row:a", "row:b", 3, 0],
40
+ });
41
+ expect(decode(wire)).toEqual(value);
42
+ // The table is consumed, so a second decode of the same object cannot remap numbers.
43
+ expect(decode(wire)).toEqual(value);
44
+ });
45
+
46
+ it("rejects an index outside the table, even in an optional slot", () => {
47
+ const wire = {
48
+ rows: [{ id: 0, count: 1, parentIds: [], mergedId: 5, item: { ownerId: null, legacyId: 0 } }],
49
+ ids: ["row:a"],
50
+ };
51
+
52
+ expect(() => decode(wire)).toThrow(/mergedId/);
53
+ });
54
+
55
+ it("sends and reads the plain form without the marker", () => {
56
+ const value = {
57
+ rows: [{ id: "row:a", count: 1, parentIds: [], item: { ownerId: null, legacyId: 1 } }],
58
+ };
59
+
60
+ expect(encode(value)).toEqual(value);
61
+ expect(decode(value)).toEqual(value);
62
+ });
63
+});
packages/contracts/src/orchestrationV2IdTable.ts
new
+125
@@ -0,0 +1,125 @@
1
+import * as Schema from "effect/Schema";
2
+import * as SchemaGetter from "effect/SchemaGetter";
3
+
4
+/**
5
+ * Opt-in wire form: on each record (top-level array element, top-level object, or `item`), values
6
+ * under `id`, `*Id`, and `*Ids` index the response's `ids` table. Both peers must apply this exact
7
+ * rule; changing it needs a new opt-in (`compactIds=2`, or a new `accept*` key over WS).
8
+ */
9
+type JsonObject = { [key: string]: unknown };
10
+
11
+const isObject = (value: unknown): value is JsonObject =>
12
+ typeof value === "object" && value !== null && !Array.isArray(value);
13
+const isIdKey = (key: string) => key === "id" || key.endsWith("Id");
14
+const isIdListKey = (key: string) => key.endsWith("Ids");
15
+
16
+function internRecordIds(container: JsonObject, ids: Array<string | number>): JsonObject {
17
+ const index = new Map<string | number, number>();
18
+ const ref = (value: unknown) => {
19
+ if (typeof value !== "string" && typeof value !== "number") return value;
20
+ let position = index.get(value);
21
+ if (position === undefined) {
22
+ position = ids.length;
23
+ ids.push(value);
24
+ index.set(value, position);
25
+ }
26
+ return position;
27
+ };
28
+ // Copies each record: unknown members can pass through encoding by reference.
29
+ const internRecord = (record: JsonObject): JsonObject => {
30
+ const out: JsonObject = {};
31
+ for (const key in record) {
32
+ const value = record[key];
33
+ out[key] = isIdKey(key)
34
+ ? ref(value)
35
+ : isIdListKey(key) && Array.isArray(value)
36
+ ? value.map(ref)
37
+ : key === "item" && isObject(value)
38
+ ? internRecord(value)
39
+ : value;
40
+ }
41
+ return out;
42
+ };
43
+ const out: JsonObject = {};
44
+ for (const key in container) {
45
+ const value = container[key];
46
+ out[key] = Array.isArray(value)
47
+ ? value.map((record) => (isObject(record) ? internRecord(record) : record))
48
+ : isObject(value)
49
+ ? internRecord(value)
50
+ : value;
51
+ }
52
+ return out;
53
+}
54
+
55
+// Mutates freshly parsed JSON in place; the client owns it and this avoids a copy.
56
+function expandRecordIds(container: JsonObject, ids: ReadonlyArray<unknown>): void {
57
+ const expandRecord = (record: JsonObject) => {
58
+ for (const key in record) {
59
+ const value = record[key];
60
+ if (typeof value === "number") {
61
+ if (isIdKey(key)) record[key] = ids[value];
62
+ } else if (Array.isArray(value)) {
63
+ if (isIdListKey(key)) {
64
+ for (let i = 0; i < value.length; i++) {
65
+ const entry = value[i];
66
+ if (typeof entry === "number") value[i] = ids[entry];
67
+ }
68
+ }
69
+ } else if (key === "item" && isObject(value)) {
70
+ expandRecord(value);
71
+ }
72
+ }
73
+ };
74
+ for (const key in container) {
75
+ const value = container[key];
76
+ if (Array.isArray(value)) {
77
+ for (const record of value) if (isObject(record)) expandRecord(record);
78
+ } else if (isObject(value)) {
79
+ expandRecord(value);
80
+ }
81
+ }
82
+}
83
+
84
+// `toCodecJson` turns `Schema.Unknown` into a recursive JSON check of the whole
85
+// tree. The input is parsed JSON and the inner codec validates it anyway.
86
+const ParsedJson = Schema.declare((_: unknown): _ is unknown => true, {
87
+ toCodecJson: () => undefined,
88
+});
89
+
90
+/**
91
+ * Adds the optional ID table to a snapshot-shaped schema. `container` returns the
92
+ * key whose value holds the records, `null` when records sit at the top level, or
93
+ * `undefined` when the value has no records (other stream item kinds).
94
+ */
95
+export const withRecordIdTable = <S extends Schema.Top>(
96
+ schema: S,
97
+ container: (value: JsonObject) => string | null | undefined,
98
+) => {
99
+ const codec = Schema.toCodecJson(schema);
100
+ type Encoded = typeof codec.Encoded;
101
+ return ParsedJson.pipe(
102
+ Schema.decodeTo(codec, {
103
+ decode: SchemaGetter.transform((raw: unknown): Encoded => {
104
+ if (!isObject(raw) || !Array.isArray(raw.ids)) return raw as Encoded;
105
+ // Consumed on the parsed object itself, so decoding it again is a no-op.
106
+ const ids: ReadonlyArray<unknown> = raw.ids;
107
+ delete raw.ids;
108
+ const key = container(raw);
109
+ const target = key === null ? raw : key === undefined ? undefined : raw[key];
110
+ if (isObject(target)) expandRecordIds(target, ids);
111
+ return raw as Encoded;
112
+ }),
113
+ encode: SchemaGetter.transform((encoded: Encoded): unknown => {
114
+ if (!isObject(encoded) || encoded.idTable !== true) return encoded;
115
+ const { idTable: _idTable, ...rest } = encoded;
116
+ const key = container(rest);
117
+ const ids: Array<string | number> = [];
118
+ if (key === null) return { ...internRecordIds(rest, ids), ids };
119
+ const target = key === undefined ? undefined : rest[key];
120
+ if (key === undefined || !isObject(target)) return rest;
121
+ return { ...rest, [key]: internRecordIds(target, ids), ids };
122
+ }),
123
+ }),
124
+ );
125
+};