@devries-cameron20 / t3code Mirror · read-only / commits / 27e0c684c5
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 18:15 UTC

perf(server): coalesce superseded streaming text updates (#24267)

Julius Marminge committed Oct 24, 2026 at 02:17 UTC 27e0c684c585f955252d65f06206dd11a143bff5
2 files changed +344 -50
apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.test.ts
+303 -5
@@ -1,10 +1,14 @@
1 import {
2 EventId,
3 MessageId,
4 + ProjectId,
5 + ProviderInstanceId,
6 + type OrchestrationV2ThreadProjection,
7 ThreadId,
8 RunId,
9 TurnItemId,
10 type OrchestrationV2ThreadStreamItem,
11 + type OrchestrationV2TurnItem,
12 } from "@t3tools/contracts";
13 import { it } from "@effect/vitest";
14 import * as DateTime from "effect/DateTime";
@@ -13,11 +17,13 @@ import * as Stream from "effect/Stream";
17 import * as TestClock from "effect/testing/TestClock";
18 import { describe, expect } from "vite-plus/test";
19 import {
16 - coalesceLiveToolUpdatedEvents,
20 + coalesceLiveUpdatedEvents,
21 coalesceThreadLiveStream,
22 makeThreadLiveEventCoalescer,
23 } from "./ThreadLiveEventCoalescer.ts";
24
25 +import { applyOrchestrationV2ProjectionEvent } from "../../../../packages/client-runtime/src/state/orchestrationV2Projection.ts";
26 +
27 type ThreadEvent = Extract<OrchestrationV2ThreadStreamItem, { readonly kind: "event" }>;
28 const threadId = ThreadId.make("thread-coalescer-test");
29 const runId = RunId.make("run-coalescer-test");
@@ -94,15 +100,69 @@ function makeMessage(sequence: number, text = "Still working"): ThreadEvent {
100 };
101 }
102
103 +function makeStreamingMessage(
104 + sequence: number,
105 + options: {
106 + readonly id?: string;
107 + readonly text?: string;
108 + readonly streaming?: boolean;
109 + } = {},
110 +): ThreadEvent {
111 + const message = makeMessage(sequence, options.text ?? `text-${sequence}`);
112 + if (message.event.type !== "message.updated") throw new Error("Expected message");
113 + return {
114 + ...message,
115 + event: {
116 + ...message.event,
117 + payload: {
118 + ...message.event.payload,
119 + id: MessageId.make(options.id ?? "message"),
120 + streaming: options.streaming ?? true,
121 + },
122 + },
123 + };
124 +}
125 +
126 +function makeTextItem(
127 + sequence: number,
128 + options: {
129 + readonly id?: string;
130 + readonly text?: string;
131 + readonly type?: "assistant_message" | "reasoning";
132 + readonly status?: OrchestrationV2TurnItem["status"];
133 + readonly streaming?: boolean;
134 + } = {},
135 +): ThreadEvent {
136 + const tool = makeToolActivity(sequence);
137 + if (tool.event.type !== "turn-item.updated" || tool.event.payload.type !== "command_execution")
138 + throw new Error("Expected command item");
139 + const { input: _input, output: _output, ...base } = tool.event.payload;
140 + return {
141 + ...tool,
142 + event: {
143 + ...tool.event,
144 + payload: {
145 + ...base,
146 + id: TurnItemId.make(options.id ?? "item"),
147 + type: options.type ?? "assistant_message",
148 + messageId: MessageId.make("message"),
149 + text: options.text ?? `text-${sequence}`,
150 + status: options.status ?? "running",
151 + streaming: options.streaming ?? true,
152 + },
153 + },
154 + };
155 +}
156 +
157 describe("ThreadLiveEventCoalescer", () => {
98 - it("coalesces only calls with a stable toolCallId", () => {
158 + it("preserves tool-call identity switches as insertion-order boundaries", () => {
159 const events = [
160 makeToolActivity(1, { toolCallId: "call-a" }),
161 makeToolActivity(2, { toolCallId: "call-b" }),
162 makeToolActivity(3, { toolCallId: "call-a" }),
163 ];
164
105 - expect(coalesceLiveToolUpdatedEvents(events).map((event) => event.sequence)).toEqual([2, 3]);
165 + expect(coalesceLiveUpdatedEvents(events).map((event) => event.sequence)).toEqual([1, 2, 3]);
166 });
167
168 it("does not coalesce stable tool calls across turns", () => {
@@ -111,7 +171,7 @@ describe("ThreadLiveEventCoalescer", () => {
171 makeToolActivity(2, { turnId: RunId.make("turn-new") }),
172 ];
173
114 - expect(coalesceLiveToolUpdatedEvents(events).map((event) => event.sequence)).toEqual([1, 2]);
174 + expect(coalesceLiveUpdatedEvents(events).map((event) => event.sequence)).toEqual([1, 2]);
175 });
176
177 it("flushes a stable update run before a completion boundary", () => {
@@ -122,9 +182,247 @@ describe("ThreadLiveEventCoalescer", () => {
182 makeToolActivity(4),
183 ];
184
125 - expect(coalesceLiveToolUpdatedEvents(events).map((event) => event.sequence)).toEqual([2, 3, 4]);
185 + expect(coalesceLiveUpdatedEvents(events).map((event) => event.sequence)).toEqual([2, 3, 4]);
186 + });
187 +
188 + it("keeps the latest full text in sequence order and preserves state after every survivor cursor", () => {
189 + const events = [
190 + makeStreamingMessage(1, { id: "shared", text: "H" }),
191 + makeTextItem(2, { id: "shared", text: "H" }),
192 + makeTextItem(3, { id: "reasoning", type: "reasoning", text: "Think" }),
193 + makeToolActivity(4),
194 + makeTextItem(5, { id: "unique", text: "Keep me" }),
195 + makeStreamingMessage(6, { id: "shared", text: "Hello" }),
196 + makeTextItem(7, { id: "shared", text: "Hello" }),
197 + makeTextItem(8, { id: "reasoning", type: "reasoning", text: "Thinking done" }),
198 + makeToolActivity(9),
199 + makeStreamingMessage(10, { id: "shared", streaming: false, text: "Hello!" }),
200 + makeTextItem(11, { id: "shared", streaming: false, status: "completed", text: "Hello!" }),
201 + ];
202 + const survivors = coalesceLiveUpdatedEvents(events);
203 + expect(survivors).toEqual(events);
204 +
205 + const latestStates = (input: ReadonlyArray<ThreadEvent>) =>
206 + Object.fromEntries(
207 + input.map(({ event }) => [
208 + `${event.type}:${"id" in event.payload ? event.payload.id : ""}`,
209 + event.payload,
210 + ]),
211 + );
212 + for (let index = 0; index < survivors.length; index += 1) {
213 + const received = survivors.slice(0, index + 1);
214 + const cursor = received.at(-1)!.sequence;
215 + const resumed = coalesceLiveUpdatedEvents(events.filter((event) => event.sequence > cursor));
216 + expect(latestStates([...received, ...resumed])).toEqual(latestStates(events));
217 + }
218 });
219
220 + it("preserves assistant-message identity changes as insertion-order boundaries", () => {
221 + const events = [
222 + makeStreamingMessage(1, { id: "message-a" }),
223 + makeTextItem(2, { id: "item-a" }),
224 + makeStreamingMessage(3, { id: "message-b" }),
225 + makeTextItem(4, { id: "item-b" }),
226 + makeStreamingMessage(5, { id: "message-a" }),
227 + makeTextItem(6, { id: "item-a" }),
228 + ];
229 + expect(coalesceLiveUpdatedEvents(events)).toEqual(events);
230 + });
231 +
232 + it.effect(
233 + "preserves turn-item insertion and partial timeline state after every survivor cursor",
234 + () =>
235 + Effect.gen(function* () {
236 + const item = (
237 + sequence: number,
238 + id: string,
239 + text: string,
240 + completed = false,
241 + ): ThreadEvent => {
242 + const input = makeTextItem(sequence, {
243 + id,
244 + text,
245 + type: "reasoning",
246 + status: completed ? "completed" : "running",
247 + streaming: !completed,
248 + });
249 + if (input.event.type !== "turn-item.updated") throw new Error("Expected turn item");
250 + return {
251 + ...input,
252 + event: {
253 + ...input.event,
254 + runId: undefined,
255 + payload: {
256 + ...input.event.payload,
257 + runId: null,
258 + ordinal: id === "a" ? 1 : 2,
259 + completedAt: completed ? now : null,
260 + },
261 + },
262 + };
263 + };
264 + const events = [
265 + item(1, "a", "A"),
266 + item(2, "b", "B"),
267 + item(3, "a", "AA"),
268 + item(4, "a", "Final A", true),
269 + item(5, "b", "Final B", true),
270 + ];
271 + const emptyProjection = {
272 + thread: {
273 + id: threadId,
274 + projectId: ProjectId.make("project-reducer"),
275 + title: "Reducer",
276 + providerInstanceId: ProviderInstanceId.make("codex"),
277 + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" },
278 + runtimeMode: "full-access",
279 + interactionMode: "default",
280 + branch: null,
281 + worktreePath: null,
282 + activeProviderThreadId: null,
283 + lineage: { rootThreadId: threadId, parentThreadId: null, relationshipToParent: null },
284 + forkedFrom: null,
285 + createdBy: "user",
286 + creationSource: "web",
287 + createdAt: now,
288 + updatedAt: now,
289 + archivedAt: null,
290 + settledOverride: null,
291 + settledAt: null,
292 + lastVisitedAt: null,
293 + deletedAt: null,
294 + },
295 + runs: [],
296 + attempts: [],
297 + nodes: [],
298 + subagents: [],
299 + providerSessions: [],
300 + providerThreads: [],
301 + providerTurns: [],
302 + runtimeRequests: [],
303 + messages: [],
304 + plans: [],
305 + turnItems: [],
306 + checkpointScopes: [],
307 + checkpoints: [],
308 + contextHandoffs: [],
309 + contextTransfers: [],
310 + visibleTurnItems: [],
311 + updatedAt: now,
312 + } satisfies OrchestrationV2ThreadProjection;
313 + const apply = (inputs: ReadonlyArray<ThreadEvent>, partialTimeline: boolean) => {
314 + let projection: OrchestrationV2ThreadProjection = emptyProjection;
315 + let latestLocalTurnOrdinal = 0;
316 + for (const { event } of inputs) {
317 + const next = applyOrchestrationV2ProjectionEvent(projection, event, {
318 + partialTimeline,
319 + latestLocalTurnOrdinal,
320 + });
321 + if (next === null || next === projection) continue;
322 + projection = next;
323 + if (partialTimeline && event.type === "turn-item.updated") {
324 + latestLocalTurnOrdinal = Math.max(latestLocalTurnOrdinal, event.payload.ordinal);
325 + }
326 + }
327 + return projection;
328 + };
329 + const survivors = yield* Stream.fromIterable(events).pipe(
330 + coalesceThreadLiveStream,
331 + Stream.filter((input): input is ThreadEvent => input.kind === "event"),
332 + Stream.runCollect,
333 + );
334 + for (const partialTimeline of [true, false]) {
335 + const expected = apply(events, partialTimeline);
336 + expect(
337 + expected.turnItems.map((item) => [item.id, "text" in item ? item.text : null]),
338 + ).toEqual([
339 + ["a", "Final A"],
340 + ["b", "Final B"],
341 + ]);
342 + expect(apply(survivors, partialTimeline)).toEqual(expected);
343 + for (const [index, survivor] of survivors.entries()) {
344 + const received = survivors.slice(0, index + 1);
345 + const replay = yield* Stream.fromIterable(
346 + events.filter((event) => event.sequence > survivor.sequence),
347 + ).pipe(
348 + coalesceThreadLiveStream,
349 + Stream.filter((input): input is ThreadEvent => input.kind === "event"),
350 + Stream.runCollect,
351 + );
352 + expect(apply([...received, ...replay], partialTimeline)).toEqual(expected);
353 + }
354 + }
355 + }),
356 + );
357 +
358 + it.effect("flushes text at synchronization and completion boundaries", () =>
359 + Effect.scoped(
360 + Effect.gen(function* () {
361 + const coalescer = yield* makeThreadLiveEventCoalescer();
362 + const events = [
363 + makeStreamingMessage(1),
364 + makeTextItem(2, { type: "reasoning" }),
365 + makeStreamingMessage(3),
366 + makeTextItem(4, { type: "reasoning" }),
367 + { kind: "synchronized" } as const,
368 + makeStreamingMessage(5),
369 + makeTextItem(6, { type: "reasoning" }),
370 + makeStreamingMessage(7, { streaming: false, text: "Final answer" }),
371 + makeTextItem(8, {
372 + type: "reasoning",
373 + streaming: false,
374 + status: "completed",
375 + text: "Final answer",
376 + }),
377 + makeStreamingMessage(9),
378 + makeTextItem(10, { type: "reasoning" }),
379 + ];
380 + yield* coalescer.offerAll(events);
381 + yield* coalescer.end;
382 + expect(yield* Stream.runCollect(coalescer.stream)).toEqual(events.slice(2));
383 + expect(yield* coalescer.usage).toEqual({ retainedItems: 0, retainedSerializedBytes: 0 });
384 + }),
385 + ),
386 + );
387 +
388 + it.effect(
389 + "flushes growing text on the fixed window and charges only unacknowledged survivors",
390 + () =>
391 + Effect.scoped(
392 + Effect.gen(function* () {
393 + const coalescer = yield* makeThreadLiveEventCoalescer();
394 + const first = [makeStreamingMessage(1, { text: "é" }), makeTextItem(2, { text: "é" })];
395 + const latest = [makeStreamingMessage(3, { text: "éé" }), makeTextItem(4, { text: "éé" })];
396 + const serializedBytes = (events: ReadonlyArray<ThreadEvent>) =>
397 + events.reduce((sum, event) => sum + Buffer.byteLength(encodeEvent(event)), 0);
398 + yield* coalescer.offerAll(first);
399 + yield* TestClock.adjust("40 millis");
400 + yield* coalescer.offerAll(latest);
401 + expect(yield* coalescer.usage).toEqual({
402 + retainedItems: 4,
403 + retainedSerializedBytes: serializedBytes([...first, ...latest]),
404 + });
405 + yield* TestClock.adjust("10 millis");
406 + const expectedUsage = {
407 + retainedItems: 2,
408 + retainedSerializedBytes: serializedBytes(latest),
409 + };
410 + expect(yield* coalescer.usage).toEqual(expectedUsage);
411 + const pull = yield* Stream.toPull(coalescer.stream);
412 + expect(yield* pull).toEqual(latest);
413 + expect(yield* coalescer.usage).toEqual(expectedUsage);
414 +
415 + const next = [makeStreamingMessage(5), makeTextItem(6)];
416 + yield* coalescer.offerAll(next);
417 + yield* TestClock.adjust("50 millis");
418 + expect(yield* pull).toEqual(next);
419 + yield* coalescer.end;
420 + yield* pull.pipe(Effect.result);
421 + expect(yield* coalescer.usage).toEqual({ retainedItems: 0, retainedSerializedBytes: 0 });
422 + }),
423 + ),
424 + );
425 +
426 it.effect("keeps the flush timer alive when an offer's shorter scope closes", () =>
427 Effect.scoped(
428 Effect.gen(function* () {
apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.ts
+41 -45
@@ -21,63 +21,59 @@ const MAX_PENDING_UPDATES = 512;
21 type ThreadLiveEvent = Extract<OrchestrationV2ThreadStreamItem, { readonly kind: "event" }>;
22 export type ThreadLiveInput = ThreadLiveEvent | { readonly kind: "synchronized" };
23
24 -function isToolUpdated(input: ThreadLiveEvent): boolean {
25 - if (input.event.type !== "turn-item.updated" || input.event.payload.status !== "running")
26 - return false;
27 - switch (input.event.payload.type) {
24 +function coalescibleUpdateKey(input: ThreadLiveEvent): string | null {
25 + const { event } = input;
26 + const prefix = `${event.threadId}\u0000${event.runId ?? ""}\u0000${event.type}`;
27 + if (event.type === "message.updated") {
28 + return event.payload.role === "assistant" && event.payload.streaming
29 + ? `${prefix}\u0000${event.payload.id}`
30 + : null;
31 + }
32 + if (event.type !== "turn-item.updated" || event.payload.status !== "running") return null;
33 + switch (event.payload.type) {
34 + case "assistant_message":
35 + case "reasoning":
36 + if (!event.payload.streaming) return null;
37 + break;
38 case "command_execution":
39 case "file_change":
40 case "file_search":
41 case "web_search":
42 case "dynamic_tool":
33 - return true;
43 + break;
44 default:
35 - return false;
45 + return null;
46 }
47 + return `${prefix}\u0000${event.payload.type}\u0000${event.payload.id}`;
48 }
49
39 -function stableToolCallIdentity(input: ThreadLiveEvent): string | null {
40 - return input.event.type === "turn-item.updated" ? input.event.payload.id : null;
41 -}
42 -
43 -export function coalesceLiveToolUpdatedEvents(
50 +// These events replace whole entities. Keep each survivor at its original
51 +// position/sequence so a resume cursor cannot skip an unrelated earlier update.
52 +export function coalesceLiveUpdatedEvents(
53 events: ReadonlyArray<ThreadLiveEvent>,
54 ): ReadonlyArray<ThreadLiveEvent> {
46 - const survivors: Array<ThreadLiveEvent> = [];
47 - let pendingUpdates: Array<ThreadLiveEvent> = [];
48 -
49 - const flushUpdates = () => {
50 - const seen = new Set<string>();
51 - const latestUpdates: Array<ThreadLiveEvent> = [];
52 - for (let index = pendingUpdates.length - 1; index >= 0; index -= 1) {
53 - const event = pendingUpdates[index]!;
54 - const identity = stableToolCallIdentity(event);
55 - const key = identity
56 - ? `${event.event.threadId}\u0000${event.event.runId ?? ""}\u0000${identity}`
57 - : null;
58 - if (key && seen.has(key)) {
59 - continue;
60 - }
61 - if (key) {
62 - seen.add(key);
63 - }
64 - latestUpdates.push(event);
65 - }
66 - latestUpdates.reverse();
67 - survivors.push(...latestUpdates);
68 - pendingUpdates = [];
69 - };
70 -
71 - for (const event of events) {
72 - if (isToolUpdated(event)) {
73 - pendingUpdates.push(event);
55 + const latestUpdates = new Map<string, number>();
56 + const superseded = new Set<number>();
57 + const pendingKeys = new Map<ThreadLiveEvent["event"]["type"], string>();
58 +
59 + for (const [index, event] of events.entries()) {
60 + const key = coalescibleUpdateKey(event);
61 + if (key === null) {
62 + latestUpdates.clear();
63 + pendingKeys.clear();
64 continue;
65 }
76 - flushUpdates();
77 - survivors.push(event);
66 + // Both entity families use insertion order; partial timelines also reject
67 + // unseen items below their watermark. Keep identity switches within each
68 + // family, while allowing paired message/item updates to coalesce.
69 + const pendingKey = pendingKeys.get(event.event.type);
70 + if (pendingKey !== undefined && pendingKey !== key) latestUpdates.clear();
71 + pendingKeys.set(event.event.type, key);
72 + const previous = latestUpdates.get(key);
73 + if (previous !== undefined) superseded.add(previous);
74 + latestUpdates.set(key, index);
75 }
79 - flushUpdates();
80 - return survivors;
76 + return events.filter((_, index) => !superseded.has(index));
77 }
78
79 export const makeThreadLiveEventCoalescer = Effect.fn(
@@ -116,7 +112,7 @@ export const makeThreadLiveEventCoalescer = Effect.fn(
112 }
113 const items = yield* budget.replace(
114 pendingUpdates,
119 - coalesceLiveToolUpdatedEvents(pendingUpdates.map((item) => item.value)),
115 + coalesceLiveUpdatedEvents(pendingUpdates.map((item) => item.value)),
116 );
117 pendingUpdates = [];
118 yield* Queue.offerAll(output, items);
@@ -156,7 +152,7 @@ export const makeThreadLiveEventCoalescer = Effect.fn(
152 Effect.uninterruptible,
153 );
154 }
159 - if (input.kind === "event" && isToolUpdated(input)) {
155 + if (input.kind === "event" && coalescibleUpdateKey(input) !== null) {
156 if (pendingUpdates.length === 1) {
157 const generation = ++windowGeneration;
158 windowFiber = yield* Effect.forkIn(flushWindow(generation), coalescerScope);