The best way to control Claude Code, Codex and more from any computer or phone
perf(server): batch terminal output that arrives within 2 ms (#37060)
Julius Marminge committed
Nov 14, 2026 at 23:33 UTC
efddcfeb20c9718b950645639aed4ace1cc79a37
2 files changed
+80
-9
apps/server/src/terminal/OutputProtocol.test.ts
+55
-6
@@ -218,6 +218,9 @@ describe("terminal output stream", () => {
218
Effect.forkScoped,
219
);
220
const write = yield* Deferred.await(receive);
221
+ // Each event arrives after the batching window, so it is sent on its own.
222
+ const publish = (event: TerminalAttachStreamEvent) =>
223
+ TestClock.adjust("2 millis").pipe(Effect.andThen(publisher.publish(event)));
224
const takeChunk = Effect.fnUntraced(function* (queue: typeof stalled) {
225
const response = yield* Queue.take(queue);
226
assert.equal(response._tag, "Chunk");
@@ -234,7 +237,7 @@ describe("terminal output stream", () => {
237
// Eight unacknowledged chunks fill only client 0's output window.
238
for (let index = 0; index < 7; index++) {
239
const event = output("before-stall-" + index);
237
- yield* publisher.publish(event);
240
+ yield* publish(event);
241
assert.deepEqual(yield* takeChunk(stalled), [event]);
242
assert.deepEqual(yield* takeChunk(healthy), [event]);
243
yield* write(1, { _tag: "Ack", requestId: "2" });
@@ -242,7 +245,7 @@ describe("terminal output stream", () => {
245
assert.equal(publisher.listeners.size, 2);
246
for (let index = 0; index < 100; index++) {
247
const event = output(index + ":" + "界".repeat(1365));
245
- yield* publisher.publish(event);
248
+ yield* publish(event);
249
assert.deepEqual(yield* takeChunk(healthy), [event]);
250
yield* write(1, { _tag: "Ack", requestId: "2" });
251
if (index === 31) assert.equal(publisher.listeners.size, 2);
@@ -257,7 +260,7 @@ describe("terminal output stream", () => {
260
yield* write(0, { _tag: "Ack", requestId: "1" });
261
}
262
const after = output("unique output after recovery");
260
- yield* publisher.publish(after);
263
+ yield* publish(after);
264
assert.deepEqual(yield* takeChunk(healthy), [after]);
265
assert.deepEqual(yield* takeChunk(stalled), [after]);
266
assert.equal(yield* Queue.size(stalled), 0);
@@ -327,7 +330,48 @@ describe("terminal output stream", () => {
330
}),
331
);
332
330
- it.effect("caps each delivered batch at the output window in UTF-8 bytes", () =>
333
+ it.effect("emits idle output immediately and batches a burst until a fixed deadline", () =>
334
+ Effect.gen(function* () {
335
+ const publisher = makePublisher();
336
+ const pull = yield* Stream.toPull(terminalOutputStream(attach(publisher, "")));
337
+ assert.deepEqual(yield* pull, [snapshot("")]);
338
+
339
+ yield* TestClock.adjust("10 millis");
340
+ yield* publisher.publish(output("a"));
341
+ assert.deepEqual(yield* pull, [output("a")]);
342
+
343
+ yield* publisher.publish(output("b"));
344
+ const burst = yield* pull.pipe(Effect.forkChild({ startImmediately: true }));
345
+ yield* TestClock.adjust("1 millis");
346
+ assert.isUndefined(burst.pollUnsafe());
347
+ // Arrivals inside the window never push the deadline back.
348
+ yield* publisher.publish(output("c"));
349
+ yield* publisher.publish(output("d"));
350
+ yield* TestClock.adjust("1 millis");
351
+ assert.deepEqual(yield* Fiber.join(burst), [output("b"), output("c"), output("d")]);
352
+ }).pipe(Effect.scoped),
353
+ );
354
+
355
+ it.effect("flushes pending output early, in order, when a lifecycle event arrives", () =>
356
+ Effect.gen(function* () {
357
+ const publisher = makePublisher();
358
+ const pull = yield* Stream.toPull(terminalOutputStream(attach(publisher, "")));
359
+ assert.deepEqual(yield* pull, [snapshot("")]);
360
+
361
+ yield* publisher.publish(output("last words"));
362
+ const waiting = yield* pull.pipe(Effect.forkChild({ startImmediately: true }));
363
+ assert.isUndefined(waiting.pollUnsafe());
364
+ const closed: TerminalAttachStreamEvent = {
365
+ type: "closed",
366
+ threadId: "thread-1",
367
+ terminalId: "default",
368
+ };
369
+ yield* publisher.publish(closed);
370
+ assert.deepEqual(yield* Fiber.join(waiting), [output("last words"), closed]);
371
+ }).pipe(Effect.scoped),
372
+ );
373
+
374
+ it.effect("flushes full batches early and caps each at the output window in UTF-8 bytes", () =>
375
Effect.gen(function* () {
376
const publisher = makePublisher();
377
const pull = yield* Stream.toPull(terminalOutputStream(attach(publisher, "")));
@@ -336,8 +380,9 @@ describe("terminal output stream", () => {
380
const data = "界".repeat(1024);
381
for (let index = 0; index < 1024; index++) yield* publisher.publish(output(data));
382
383
+ // Twenty 3 KiB events fill a 64 KiB batch, so full batches skip the deadline.
384
let delivered = 0;
340
- while (delivered < 1024) {
385
+ while (delivered < 1020) {
386
const batch = yield* pull;
387
assert.isAtMost(
388
batch.reduce(
@@ -348,7 +393,11 @@ describe("terminal output stream", () => {
393
);
394
delivered += batch.length;
395
}
351
- assert.equal(delivered, 1024);
396
+ assert.equal(delivered, 1020);
397
+ const tail = yield* pull.pipe(Effect.forkChild({ startImmediately: true }));
398
+ assert.isUndefined(tail.pollUnsafe());
399
+ yield* TestClock.adjust("2 millis");
400
+ assert.lengthOf(yield* Fiber.join(tail), 4);
401
}),
402
);
403
apps/server/src/terminal/OutputProtocol.ts
+25
-3
@@ -6,6 +6,8 @@ import {
6
} from "@t3tools/contracts";
7
import type * as Arr from "effect/Array";
8
import * as Cause from "effect/Cause";
9
+import * as Clock from "effect/Clock";
10
+import * as Duration from "effect/Duration";
11
import * as Effect from "effect/Effect";
12
import * as Latch from "effect/Latch";
13
import * as Stream from "effect/Stream";
@@ -137,6 +139,9 @@ export function withOutputWindow(
139
}
140
141
const MAX_QUEUED_OUTPUT = 8 * 1024 * 1024;
142
+// Output that follows an emission this closely waits and rides along in one batch,
143
+// so a PTY that delivers one line per callback (ConPTY) is not sent one frame per line.
144
+const OUTPUT_WINDOW_NANOS = 2_000_000n;
145
146
type Listener<A> = (event: A) => Effect.Effect<void>;
147
// Acquisition must register cleanup synchronously with installing the listener.
@@ -152,11 +157,15 @@ export function terminalOutputStream<A extends TerminalEvent | TerminalAttachStr
157
): Stream.Stream<A, E> {
158
return Stream.unwrap(
159
Effect.gen(function* () {
160
+ const clock = yield* Clock.Clock;
161
const ready = yield* Latch.make();
162
let pending: Array<A | undefined> = [];
163
let sizes: number[] = [];
164
let head = 0;
165
let queued = 0;
166
+ // Pending events other than output; any of them flushes the window early.
167
+ let lifecycle = 0;
168
+ let deadline = 0n;
169
let overflowed = false;
170
let subscription = 0;
171
let unsubscribe: (() => void) | null = null;
@@ -168,7 +177,9 @@ export function terminalOutputStream<A extends TerminalEvent | TerminalAttachStr
177
sizes = [];
178
head = 0;
179
queued = 0;
180
+ lifecycle = 0;
181
};
182
+ const urgent = () => lifecycle > 0 || queued >= MAX_PENDING_BYTES;
183
yield* Effect.addFinalizer(() => Effect.sync(release));
184
185
const listen = (from: Subscribe<A, E>) =>
@@ -178,16 +189,19 @@ export function terminalOutputStream<A extends TerminalEvent | TerminalAttachStr
189
(event) =>
190
Effect.sync(() => {
191
if (id !== subscription) return;
192
+ const wasEmpty = head === pending.length;
193
const size = queuedSize(event);
194
pending.push(event);
195
sizes.push(size);
196
queued += size;
197
+ if (event.type !== "output") lifecycle += 1;
198
// A single snapshot must fit even when history fills the budget.
199
if (queued - sizes[head]! > maxQueued) {
200
release();
201
overflowed = true;
202
}
190
- ready.openUnsafe();
203
+ // The current or next pull handles the pending output deadline.
204
+ if (wasEmpty || overflowed || urgent()) ready.openUnsafe();
205
}),
206
(stop) => {
207
// Overflow or cancellation may invalidate acquisition before registration.
@@ -204,6 +218,7 @@ export function terminalOutputStream<A extends TerminalEvent | TerminalAttachStr
218
const event = pending[head]!;
219
pending[head] = undefined;
220
head += 1;
221
+ if (event.type !== "output") lifecycle -= 1;
222
return event;
223
};
224
const takeBatch = (): Arr.NonEmptyReadonlyArray<A> => {
@@ -228,9 +243,16 @@ export function terminalOutputStream<A extends TerminalEvent | TerminalAttachStr
243
overflowed = false;
244
return resync ? listen(resync).pipe(Effect.andThen(pull)) : Cause.done();
245
}
231
- if (head < pending.length) return Effect.succeed(takeBatch());
246
ready.closeUnsafe();
233
- return ready.await.pipe(Effect.andThen(pull));
247
+ if (head === pending.length) return ready.await.pipe(Effect.andThen(pull));
248
+ const now = clock.monotonicTimeNanosUnsafe();
249
+ if (now >= deadline || urgent()) {
250
+ deadline = now + OUTPUT_WINDOW_NANOS;
251
+ return Effect.succeed(takeBatch());
252
+ }
253
+ return Effect.raceFirst(ready.await, Effect.sleep(Duration.nanos(deadline - now))).pipe(
254
+ Effect.andThen(pull),
255
+ );
256
},
257
);
258