@devries-cameron20 / t3code-1 / commits / efddcfeb20
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

perf(server): batch terminal output that arrives within 2 ms (#37060)

Julius Marminge committed Nov 14, 2026 at 23:33 UTC efddcfeb20c9718b950645639aed4ace1cc79a37
2 files changed +80 -9
apps/server/src/terminal/OutputProtocol.test.ts
+55 -6
@@ -218,6 +218,9 @@ describe("terminal output stream", () => {
218 Effect.forkScoped,
219 );
220 const write = yield* Deferred.await(receive);
221 + // Each event arrives after the batching window, so it is sent on its own.
222 + const publish = (event: TerminalAttachStreamEvent) =>
223 + TestClock.adjust("2 millis").pipe(Effect.andThen(publisher.publish(event)));
224 const takeChunk = Effect.fnUntraced(function* (queue: typeof stalled) {
225 const response = yield* Queue.take(queue);
226 assert.equal(response._tag, "Chunk");
@@ -234,7 +237,7 @@ describe("terminal output stream", () => {
237 // Eight unacknowledged chunks fill only client 0's output window.
238 for (let index = 0; index < 7; index++) {
239 const event = output("before-stall-" + index);
237 - yield* publisher.publish(event);
240 + yield* publish(event);
241 assert.deepEqual(yield* takeChunk(stalled), [event]);
242 assert.deepEqual(yield* takeChunk(healthy), [event]);
243 yield* write(1, { _tag: "Ack", requestId: "2" });
@@ -242,7 +245,7 @@ describe("terminal output stream", () => {
245 assert.equal(publisher.listeners.size, 2);
246 for (let index = 0; index < 100; index++) {
247 const event = output(index + ":" + "界".repeat(1365));
245 - yield* publisher.publish(event);
248 + yield* publish(event);
249 assert.deepEqual(yield* takeChunk(healthy), [event]);
250 yield* write(1, { _tag: "Ack", requestId: "2" });
251 if (index === 31) assert.equal(publisher.listeners.size, 2);
@@ -257,7 +260,7 @@ describe("terminal output stream", () => {
260 yield* write(0, { _tag: "Ack", requestId: "1" });
261 }
262 const after = output("unique output after recovery");
260 - yield* publisher.publish(after);
263 + yield* publish(after);
264 assert.deepEqual(yield* takeChunk(healthy), [after]);
265 assert.deepEqual(yield* takeChunk(stalled), [after]);
266 assert.equal(yield* Queue.size(stalled), 0);
@@ -327,7 +330,48 @@ describe("terminal output stream", () => {
330 }),
331 );
332
330 - it.effect("caps each delivered batch at the output window in UTF-8 bytes", () =>
333 + it.effect("emits idle output immediately and batches a burst until a fixed deadline", () =>
334 + Effect.gen(function* () {
335 + const publisher = makePublisher();
336 + const pull = yield* Stream.toPull(terminalOutputStream(attach(publisher, "")));
337 + assert.deepEqual(yield* pull, [snapshot("")]);
338 +
339 + yield* TestClock.adjust("10 millis");
340 + yield* publisher.publish(output("a"));
341 + assert.deepEqual(yield* pull, [output("a")]);
342 +
343 + yield* publisher.publish(output("b"));
344 + const burst = yield* pull.pipe(Effect.forkChild({ startImmediately: true }));
345 + yield* TestClock.adjust("1 millis");
346 + assert.isUndefined(burst.pollUnsafe());
347 + // Arrivals inside the window never push the deadline back.
348 + yield* publisher.publish(output("c"));
349 + yield* publisher.publish(output("d"));
350 + yield* TestClock.adjust("1 millis");
351 + assert.deepEqual(yield* Fiber.join(burst), [output("b"), output("c"), output("d")]);
352 + }).pipe(Effect.scoped),
353 + );
354 +
355 + it.effect("flushes pending output early, in order, when a lifecycle event arrives", () =>
356 + Effect.gen(function* () {
357 + const publisher = makePublisher();
358 + const pull = yield* Stream.toPull(terminalOutputStream(attach(publisher, "")));
359 + assert.deepEqual(yield* pull, [snapshot("")]);
360 +
361 + yield* publisher.publish(output("last words"));
362 + const waiting = yield* pull.pipe(Effect.forkChild({ startImmediately: true }));
363 + assert.isUndefined(waiting.pollUnsafe());
364 + const closed: TerminalAttachStreamEvent = {
365 + type: "closed",
366 + threadId: "thread-1",
367 + terminalId: "default",
368 + };
369 + yield* publisher.publish(closed);
370 + assert.deepEqual(yield* Fiber.join(waiting), [output("last words"), closed]);
371 + }).pipe(Effect.scoped),
372 + );
373 +
374 + it.effect("flushes full batches early and caps each at the output window in UTF-8 bytes", () =>
375 Effect.gen(function* () {
376 const publisher = makePublisher();
377 const pull = yield* Stream.toPull(terminalOutputStream(attach(publisher, "")));
@@ -336,8 +380,9 @@ describe("terminal output stream", () => {
380 const data = "界".repeat(1024);
381 for (let index = 0; index < 1024; index++) yield* publisher.publish(output(data));
382
383 + // Twenty 3 KiB events fill a 64 KiB batch, so full batches skip the deadline.
384 let delivered = 0;
340 - while (delivered < 1024) {
385 + while (delivered < 1020) {
386 const batch = yield* pull;
387 assert.isAtMost(
388 batch.reduce(
@@ -348,7 +393,11 @@ describe("terminal output stream", () => {
393 );
394 delivered += batch.length;
395 }
351 - assert.equal(delivered, 1024);
396 + assert.equal(delivered, 1020);
397 + const tail = yield* pull.pipe(Effect.forkChild({ startImmediately: true }));
398 + assert.isUndefined(tail.pollUnsafe());
399 + yield* TestClock.adjust("2 millis");
400 + assert.lengthOf(yield* Fiber.join(tail), 4);
401 }),
402 );
403
apps/server/src/terminal/OutputProtocol.ts
+25 -3
@@ -6,6 +6,8 @@ import {
6 } from "@t3tools/contracts";
7 import type * as Arr from "effect/Array";
8 import * as Cause from "effect/Cause";
9 +import * as Clock from "effect/Clock";
10 +import * as Duration from "effect/Duration";
11 import * as Effect from "effect/Effect";
12 import * as Latch from "effect/Latch";
13 import * as Stream from "effect/Stream";
@@ -137,6 +139,9 @@ export function withOutputWindow(
139 }
140
141 const MAX_QUEUED_OUTPUT = 8 * 1024 * 1024;
142 +// Output that follows an emission this closely waits and rides along in one batch,
143 +// so a PTY that delivers one line per callback (ConPTY) is not sent one frame per line.
144 +const OUTPUT_WINDOW_NANOS = 2_000_000n;
145
146 type Listener<A> = (event: A) => Effect.Effect<void>;
147 // Acquisition must register cleanup synchronously with installing the listener.
@@ -152,11 +157,15 @@ export function terminalOutputStream<A extends TerminalEvent | TerminalAttachStr
157 ): Stream.Stream<A, E> {
158 return Stream.unwrap(
159 Effect.gen(function* () {
160 + const clock = yield* Clock.Clock;
161 const ready = yield* Latch.make();
162 let pending: Array<A | undefined> = [];
163 let sizes: number[] = [];
164 let head = 0;
165 let queued = 0;
166 + // Pending events other than output; any of them flushes the window early.
167 + let lifecycle = 0;
168 + let deadline = 0n;
169 let overflowed = false;
170 let subscription = 0;
171 let unsubscribe: (() => void) | null = null;
@@ -168,7 +177,9 @@ export function terminalOutputStream<A extends TerminalEvent | TerminalAttachStr
177 sizes = [];
178 head = 0;
179 queued = 0;
180 + lifecycle = 0;
181 };
182 + const urgent = () => lifecycle > 0 || queued >= MAX_PENDING_BYTES;
183 yield* Effect.addFinalizer(() => Effect.sync(release));
184
185 const listen = (from: Subscribe<A, E>) =>
@@ -178,16 +189,19 @@ export function terminalOutputStream<A extends TerminalEvent | TerminalAttachStr
189 (event) =>
190 Effect.sync(() => {
191 if (id !== subscription) return;
192 + const wasEmpty = head === pending.length;
193 const size = queuedSize(event);
194 pending.push(event);
195 sizes.push(size);
196 queued += size;
197 + if (event.type !== "output") lifecycle += 1;
198 // A single snapshot must fit even when history fills the budget.
199 if (queued - sizes[head]! > maxQueued) {
200 release();
201 overflowed = true;
202 }
190 - ready.openUnsafe();
203 + // The current or next pull handles the pending output deadline.
204 + if (wasEmpty || overflowed || urgent()) ready.openUnsafe();
205 }),
206 (stop) => {
207 // Overflow or cancellation may invalidate acquisition before registration.
@@ -204,6 +218,7 @@ export function terminalOutputStream<A extends TerminalEvent | TerminalAttachStr
218 const event = pending[head]!;
219 pending[head] = undefined;
220 head += 1;
221 + if (event.type !== "output") lifecycle -= 1;
222 return event;
223 };
224 const takeBatch = (): Arr.NonEmptyReadonlyArray<A> => {
@@ -228,9 +243,16 @@ export function terminalOutputStream<A extends TerminalEvent | TerminalAttachStr
243 overflowed = false;
244 return resync ? listen(resync).pipe(Effect.andThen(pull)) : Cause.done();
245 }
231 - if (head < pending.length) return Effect.succeed(takeBatch());
246 ready.closeUnsafe();
233 - return ready.await.pipe(Effect.andThen(pull));
247 + if (head === pending.length) return ready.await.pipe(Effect.andThen(pull));
248 + const now = clock.monotonicTimeNanosUnsafe();
249 + if (now >= deadline || urgent()) {
250 + deadline = now + OUTPUT_WINDOW_NANOS;
251 + return Effect.succeed(takeBatch());
252 + }
253 + return Effect.raceFirst(ready.await, Effect.sleep(Duration.nanos(deadline - now))).pipe(
254 + Effect.andThen(pull),
255 + );
256 },
257 );
258