The best way to control Claude Code, Codex and more from any computer or phone
fix(server): show idle paragraphs before the assistant turn ends (#28221)
Held assistant paragraphs now appear after 800 ms without new text while the turn remains open. Idle writes preserve ingestion order, routing and finalization. Includes lean edge-case coverage and regressions for finalized-root reopening and early timer wakes. Verified with focused tests, check:changed, real web and compiled macOS captures, independent Codex review, and a fresh Claude Opus 5.5 READY review. Natural Antigravity hangs remain unverified. Implemented with Claude Opus 5.5 through Claude Code; investigated and verified with GPT-6.1-Sol through Codex in T3 Code.
shivam committed
Nov 10, 2026 at 10:05 UTC
e89abadaee70e0d73fb19a5ce40a686b1a92310e
6 files changed
+929
-119
apps/server/src/orchestration-v2/EventSink.ts
+4
-4
@@ -12,7 +12,6 @@ import {
12
type ProjectId,
13
ThreadId,
14
} from "@t3tools/contracts";
15
-import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
15
import * as ProviderCommitBatch from "./ProviderCommitBatch.ts";
16
import * as Context from "effect/Context";
17
import * as DateTime from "effect/DateTime";
@@ -79,9 +78,10 @@ const CurrentProviderBatch = Context.Reference<ProviderBatchPublication | undefi
78
);
79
80
export interface EventSinkV2Shape {
82
- readonly consumeProviderEvents: <E, R, E2, R2>(
83
- source: Stream.Stream<ProviderAdapter.ProviderAdapterV2Event, E, R>,
84
- process: (event: ProviderAdapter.ProviderAdapterV2Event) => Effect.Effect<void, E2, R2>,
81
+ /** Items that are not provider snapshots, such as a consumer's own control items, never batch. */
82
+ readonly consumeProviderEvents: <A extends { readonly type: string }, E, R, E2, R2>(
83
+ source: Stream.Stream<A, E, R>,
84
+ process: (event: A) => Effect.Effect<void, E2, R2>,
85
after: Effect.Effect<boolean, E2, R2>,
86
canBatch: Effect.Effect<boolean, never, R2>,
87
) => Effect.Effect<
apps/server/src/orchestration-v2/ProviderCommitBatch.ts
+4
-6
@@ -1,4 +1,3 @@
1
-import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
1
import * as Cause from "effect/Cause";
2
import * as Clock from "effect/Clock";
3
import * as Config from "effect/Config";
@@ -19,8 +18,7 @@ export const enabled = Config.Boolean("T3CODE_PROVIDER_COMMIT_BATCHING").pipe(
18
Effect.orDie,
19
);
20
22
-type Event = ProviderAdapter.ProviderAdapterV2Event;
23
-const isSnapshot = (event: Event) =>
21
+const isSnapshot = (event: { readonly type: string }) =>
22
event.type === "node.updated" ||
23
event.type === "message.updated" ||
24
event.type === "turn_item.updated" ||
@@ -232,9 +230,9 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
230
),
231
);
232
});
235
- const consume = Effect.fnUntraced(function* <E, R, E2, R2>(
236
- source: Stream.Stream<Event, E, R>,
237
- process: (event: Event) => Effect.Effect<void, E2, R2>,
233
+ const consume = Effect.fnUntraced(function* <A extends { readonly type: string }, E, R, E2, R2>(
234
+ source: Stream.Stream<A, E, R>,
235
+ process: (event: A) => Effect.Effect<void, E2, R2>,
236
after: Effect.Effect<boolean, E2, R2>,
237
canBatch: Effect.Effect<boolean, never, R2> = Effect.succeed(true),
238
) {
apps/server/src/orchestration-v2/RunExecutionService.test.ts
+479
-8
@@ -14,6 +14,7 @@ import {
14
type OrchestrationV2ProviderTurn,
15
type OrchestrationV2Run,
16
type OrchestrationV2RunAttempt,
17
+ type OrchestrationV2StoredEvent,
18
type OrchestrationV2Subagent,
19
type OrchestrationV2TurnItem,
20
ProviderDriverKind,
@@ -28,14 +29,19 @@ import {
29
TurnItemId,
30
} from "@t3tools/contracts";
31
import * as Cause from "effect/Cause";
32
+import * as Clock from "effect/Clock";
33
import * as Exit from "effect/Exit";
34
+import * as Fiber from "effect/Fiber";
35
import * as DateTime from "effect/DateTime";
36
import * as Deferred from "effect/Deferred";
37
+import * as Duration from "effect/Duration";
38
import * as Effect from "effect/Effect";
39
import * as Layer from "effect/Layer";
40
import * as Option from "effect/Option";
41
+import * as Queue from "effect/Queue";
42
import * as Ref from "effect/Ref";
43
import * as Stream from "effect/Stream";
44
+import * as TestClock from "effect/testing/TestClock";
45
import { CodexAppServerRequestError } from "effect-codex-app-server/errors";
46
import { SqlError, classifySqliteError } from "effect/sql/SqlError";
47
@@ -47,6 +53,7 @@ import * as EventSink from "./EventSink.ts";
53
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
54
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
55
import * as ProjectionStore from "./ProjectionStore.ts";
56
+import * as ProviderCommitBatch from "./ProviderCommitBatch.ts";
57
import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
58
import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts";
59
import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts";
@@ -1885,6 +1892,40 @@ it.effect("keeps ingesting until queued background work clears after the probe g
1892
}),
1893
);
1894
1895
+it.effect("keeps ingesting while the last stored root snapshot reports pending work", () =>
1896
+ Effect.gen(function* () {
1897
+ // The live probe is already idle, as when it reads adapter state ahead of the queued events.
1898
+ const consumed = yield* Deferred.make<void>();
1899
+ const result = yield* Effect.raceFirst(
1900
+ runBackgroundItemScenario(
1901
+ "root-snapshot-pending-work",
1902
+ (ids) => {
1903
+ const snapshot = (pendingBackgroundWork: boolean) =>
1904
+ ({
1905
+ type: "provider_thread.updated",
1906
+ driver,
1907
+ providerThread: { id: ids.providerThreadId, appThreadId: ids.threadId },
1908
+ pendingBackgroundWork,
1909
+ }) as ProviderAdapter.ProviderAdapterV2Event;
1910
+ return [
1911
+ snapshot(true),
1912
+ rootTerminalEvent(ids, "completed"),
1913
+ backgroundTurnItemEvent(ids, "dynamic_tool", "completed", 1),
1914
+ snapshot(false),
1915
+ ];
1916
+ },
1917
+ {
1918
+ keepEventStreamOpen: true,
1919
+ hasPendingBackgroundWorkForThread: () => Effect.succeed(false),
1920
+ onEventsConsumed: Deferred.succeed(consumed, undefined).pipe(Effect.asVoid),
1921
+ },
1922
+ ).pipe(Effect.asSome),
1923
+ Deferred.await(consumed).pipe(Effect.as(Option.none())),
1924
+ );
1925
+ assert.deepEqual(result, Option.some(["root-finalized", "turn_item:completed"]));
1926
+ }),
1927
+);
1928
+
1929
it.effect("keeps ingesting a child thread's late background item completion", () =>
1930
Effect.gen(function* () {
1931
const observed = yield* runBackgroundItemScenario("bg-child-item", (ids) => [
@@ -3692,8 +3733,18 @@ function rootTerminalEvent(
3733
3734
function runBackgroundItemScenario(
3735
key: string,
3695
- makeEvents: (ids: BackgroundScenarioIds) => ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>,
3736
+ makeEvents: (
3737
+ ids: BackgroundScenarioIds,
3738
+ ) =>
3739
+ | ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>
3740
+ | Stream.Stream<ProviderAdapter.ProviderAdapterV2Event>,
3741
options?: {
3742
+ readonly onIngest?: (
3743
+ event: ProviderAdapter.ProviderAdapterV2Event,
3744
+ write: { readonly inSharedCommit: boolean },
3745
+ ) => Effect.Effect<void>;
3746
+ // Consumes through the real commit coordinator, as the event sink does.
3747
+ readonly batched?: boolean;
3748
readonly keepEventStreamOpen?: boolean;
3749
readonly hasPendingBackgroundWorkForThread?: (
3750
providerThread: OrchestrationV2ProviderThread,
@@ -3719,6 +3770,24 @@ function runBackgroundItemScenario(
3770
const previewActivity = yield* PreviewAgentActivity.PreviewAgentActivity.pipe(
3771
Effect.provide(PreviewAgentActivity.layer),
3772
);
3773
+ let inSharedCommit = false;
3774
+ const coordinator =
3775
+ options?.batched === true
3776
+ ? yield* ProviderCommitBatch.make(
3777
+ (work) =>
3778
+ Effect.suspend(() => {
3779
+ inSharedCommit = true;
3780
+ return work;
3781
+ }).pipe(
3782
+ Effect.ensuring(
3783
+ Effect.sync(() => {
3784
+ inSharedCommit = false;
3785
+ }),
3786
+ ),
3787
+ ),
3788
+ true,
3789
+ )
3790
+ : undefined;
3791
const layerTest = RunExecutionService.layer.pipe(
3792
Layer.provide(
3793
Layer.mergeAll(
@@ -3727,6 +3796,7 @@ function runBackgroundItemScenario(
3796
Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
3797
layerTurnItemPositionsTest,
3798
mockEventSink({
3799
+ ...(coordinator === undefined ? {} : { consumeProviderEvents: coordinator.consume }),
3800
write: () => Effect.succeed([]),
3801
writeWithEffects: (input) =>
3802
Effect.gen(function* () {
@@ -3744,9 +3814,10 @@ function runBackgroundItemScenario(
3814
}),
3815
IdAllocator.layer,
3816
Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
3747
- ingestNormalized: (input) =>
3748
- Effect.gen(function* () {
3817
+ ingestNormalized: (input) => {
3818
+ const write = Effect.gen(function* () {
3819
const event = input.event;
3820
+ yield* options?.onIngest?.(event, { inSharedCommit }) ?? Effect.void;
3821
if (event.type === "turn_item.updated") {
3822
yield* Ref.update(observed, (current) => [
3823
...current,
@@ -3759,8 +3830,14 @@ function runBackgroundItemScenario(
3830
`subagent:${event.subagent.status}`,
3831
]);
3832
}
3762
- return [];
3763
- }),
3833
+ // Only the count of stored events matters to the run.
3834
+ return event.type === "provider_thread.updated"
3835
+ ? [{} as OrchestrationV2StoredEvent]
3836
+ : [];
3837
+ });
3838
+ // A write outside a batch waits for queued snapshots, as the event sink's does.
3839
+ return coordinator === undefined ? write : coordinator.barrier(write);
3840
+ },
3841
}),
3842
ServerSettings.layerTest(),
3843
),
@@ -3780,7 +3857,8 @@ function runBackgroundItemScenario(
3857
: { hasPendingBackgroundWorkForThread: options.hasPendingBackgroundWorkForThread }),
3858
subscribeEvents: Effect.gen(function* () {
3859
yield* options?.onSubscribe ?? Effect.void;
3783
- const events = Stream.fromIterable(makeEvents(ids)).pipe(
3860
+ const source = makeEvents(ids);
3861
+ const events = (Stream.isStream(source) ? source : Stream.fromIterable(source)).pipe(
3862
Stream.concat(
3863
Stream.fromEffect(
3864
Effect.andThen(
@@ -3847,8 +3925,9 @@ function runBackgroundItemScenario(
3925
});
3926
}).pipe(Effect.provide(layerTest));
3927
3850
- const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds"));
3851
- assert.isTrue(Option.isSome(closed), "event ingestion fiber did not finish");
3928
+ // No timeout: tests advance the TestClock, which would fire it early.
3929
+ yield* Deferred.await(ingestionDone);
3930
+
3931
return yield* Ref.get(observed);
3932
});
3933
}
@@ -3900,3 +3979,395 @@ it.effect("keeps a failed root's still-running native children live while the ru
3979
);
3980
}),
3981
);
3982
+
3983
+it.effect.each([
3984
+ { name: "on time", wakesEarly: false },
3985
+ // Timers run on the monotonic clock, so one can fire while the wall clock still reads a
3986
+ // millisecond before its deadline.
3987
+ { name: "before the wall clock reaches the deadline", wakesEarly: true },
3988
+])("delivers held text while an open-prompt provider idles, waking $name", ({ wakesEarly }) =>
3989
+ Effect.gen(function* () {
3990
+ const key = `idle-paragraph-${wakesEarly}`;
3991
+ const ids = backgroundScenarioIds(key);
3992
+ const now = yield* DateTime.now;
3993
+ const providerEvents = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
3994
+ const delivered = yield* Ref.make<ReadonlyArray<string>>([]);
3995
+ const paragraphDelivered = yield* Deferred.make<void>();
3996
+ const nextParagraphDelivered = yield* Deferred.make<void>();
3997
+ const clock = yield* Clock.Clock;
3998
+ const assistantText = (text: string, streaming: boolean) =>
3999
+ [
4000
+ {
4001
+ type: "message.updated",
4002
+ driver,
4003
+ message: {
4004
+ createdBy: "agent",
4005
+ creationSource: "provider",
4006
+ id: MessageId.make(`message:${key}`),
4007
+ threadId: ids.threadId,
4008
+ runId: ids.runId,
4009
+ nodeId: ids.rootNodeId,
4010
+ role: "assistant",
4011
+ text,
4012
+ attachments: [],
4013
+ streaming,
4014
+ createdAt: now,
4015
+ updatedAt: now,
4016
+ },
4017
+ },
4018
+ {
4019
+ type: "turn_item.updated",
4020
+ driver,
4021
+ turnItem: {
4022
+ ...makeLinkedChildTurnItemFixture({ ids, driver, type: "assistant_message" }),
4023
+ id: ids.itemId,
4024
+ threadId: ids.threadId,
4025
+ runId: ids.runId,
4026
+ nodeId: ids.rootNodeId,
4027
+ providerTurnId: ids.rootProviderTurnId,
4028
+ text,
4029
+ streaming,
4030
+ },
4031
+ },
4032
+ ] as ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>;
4033
+
4034
+ const blockerStarted = yield* Deferred.make<void>();
4035
+ const releaseBlocker = yield* Deferred.make<void>();
4036
+ const blocker = streamedMessageEvent({
4037
+ id: `message:${key}:blocker`,
4038
+ threadId: ids.threadId,
4039
+ runId: ids.runId,
4040
+ role: "assistant",
4041
+ text: "Other",
4042
+ streaming: false,
4043
+ });
4044
+ const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(providerEvents), {
4045
+ batched: true,
4046
+ onIngest: (event, write) =>
4047
+ event.type === "message.updated" && event.message.text === "Other"
4048
+ ? Deferred.succeed(blockerStarted, undefined).pipe(
4049
+ Effect.andThen(Deferred.await(releaseBlocker)),
4050
+ )
4051
+ : event.type === "message.updated" || event.type === "turn_item.updated"
4052
+ ? Effect.gen(function* () {
4053
+ const message = event.type === "message.updated" ? event.message : event.turnItem;
4054
+ const text = "text" in message ? message.text : "";
4055
+ const streaming = "streaming" in message && message.streaming;
4056
+ yield* Ref.update(delivered, (current) => [
4057
+ ...current,
4058
+ `${event.type}:${streaming ? "streaming" : "final"}:${JSON.stringify(text)}${
4059
+ write.inSharedCommit ? " in commit" : ""
4060
+ }`,
4061
+ ]);
4062
+ if (event.type === "turn_item.updated") {
4063
+ yield* Deferred.succeed(paragraphDelivered, undefined);
4064
+ if (text === "First.\n\nWaiting on the build.\n\n")
4065
+ yield* Deferred.succeed(nextParagraphDelivered, undefined);
4066
+ }
4067
+ })
4068
+ : Effect.void,
4069
+ }).pipe(
4070
+ Effect.provideService(Clock.Clock, {
4071
+ ...clock,
4072
+ sleep: (duration) =>
4073
+ clock.sleep(wakesEarly ? Duration.subtract(duration, Duration.millis(1)) : duration),
4074
+ }),
4075
+ Effect.forkChild,
4076
+ );
4077
+ // Waking a sleep runs ingestion until it suspends, so an idle write has landed by now.
4078
+ const lastShown = Ref.get(delivered).pipe(Effect.map((shown) => shown.at(-1)));
4079
+
4080
+ yield* Queue.offerAll(providerEvents, assistantText("First.\n\nWaiting on the build", true));
4081
+ yield* Deferred.await(paragraphDelivered);
4082
+ // The provider keeps its prompt open while a backgrounded command runs.
4083
+ yield* TestClock.adjust("2 seconds");
4084
+ assert.strictEqual(
4085
+ yield* lastShown,
4086
+ 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build"',
4087
+ );
4088
+ // The next paragraph's tail still flushes on a later pause.
4089
+ yield* Queue.offerAll(
4090
+ providerEvents,
4091
+ assistantText("First.\n\nWaiting on the build.\n\nRunning", true),
4092
+ );
4093
+ yield* Deferred.await(nextParagraphDelivered);
4094
+ yield* TestClock.adjust("1 second");
4095
+ assert.strictEqual(
4096
+ yield* lastShown,
4097
+ 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\nRunning"',
4098
+ );
4099
+ yield* Queue.offerAll(providerEvents, [
4100
+ ...assistantText("First.\n\nWaiting on the build.\n\nRunning tests", true),
4101
+ blocker,
4102
+ ...assistantText("First.\n\nWaiting on the build.\n\nRunning tests passed.", false),
4103
+ ]);
4104
+ yield* Deferred.await(blockerStarted);
4105
+ // The appended text comes due while its final snapshots wait behind the blocked write.
4106
+ yield* TestClock.adjust("1 second");
4107
+ yield* Deferred.succeed(releaseBlocker, undefined);
4108
+ yield* Queue.offer(providerEvents, rootTerminalEvent(ids, "completed"));
4109
+ yield* Fiber.join(scenario);
4110
+
4111
+ // Snapshots share the batch commit; the idle write commits alone, before it counts as shown.
4112
+ // The final snapshots win over the stale append that came due behind them.
4113
+ assert.deepEqual(yield* Ref.get(delivered), [
4114
+ `message.updated:streaming:"First.\\n\\n" in commit`,
4115
+ `turn_item.updated:streaming:"First.\\n\\n" in commit`,
4116
+ 'message.updated:streaming:"First.\\n\\nWaiting on the build"',
4117
+ 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build"',
4118
+ `message.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\n" in commit`,
4119
+ `turn_item.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\n" in commit`,
4120
+ 'message.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\nRunning"',
4121
+ 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\nRunning"',
4122
+ `message.updated:final:"First.\\n\\nWaiting on the build.\\n\\nRunning tests passed." in commit`,
4123
+ `turn_item.updated:final:"First.\\n\\nWaiting on the build.\\n\\nRunning tests passed." in commit`,
4124
+ ]);
4125
+ }),
4126
+);
4127
+
4128
+const heldChildText = "partial response";
4129
+
4130
+function childAssistantSnapshot(
4131
+ ids: BackgroundScenarioIds,
4132
+): ProviderAdapter.ProviderAdapterV2Event {
4133
+ return {
4134
+ type: "turn_item.updated",
4135
+ driver,
4136
+ turnItem: {
4137
+ ...makeLinkedChildTurnItemFixture({ ids, driver, type: "assistant_message" }),
4138
+ text: heldChildText,
4139
+ },
4140
+ } as ProviderAdapter.ProviderAdapterV2Event;
4141
+}
4142
+
4143
+function linkedSubagentEvent(
4144
+ ids: BackgroundScenarioIds,
4145
+ status: "running" | "completed",
4146
+): ProviderAdapter.ProviderAdapterV2Event {
4147
+ return {
4148
+ type: "subagent.updated",
4149
+ driver,
4150
+ subagent: {
4151
+ ...makeRunOwnedSubagentFixture({
4152
+ ids,
4153
+ providerInstanceId: ProviderInstanceId.make("codex"),
4154
+ childThreadId: ids.childThreadId,
4155
+ driver,
4156
+ status: "running",
4157
+ }),
4158
+ status,
4159
+ },
4160
+ };
4161
+}
4162
+
4163
+const isHeldChildSnapshot = (event: ProviderAdapter.ProviderAdapterV2Event) =>
4164
+ event.type === "turn_item.updated" &&
4165
+ "text" in event.turnItem &&
4166
+ event.turnItem.text === heldChildText;
4167
+
4168
+// Feeds a held child snapshot, then waits until ingestion has read past it.
4169
+const holdChildSnapshot = Effect.fn(function* (
4170
+ ids: BackgroundScenarioIds,
4171
+ events: Queue.Enqueue<ProviderAdapter.ProviderAdapterV2Event, Cause.Done>,
4172
+ subagentIngested: Deferred.Deferred<void>,
4173
+) {
4174
+ yield* Queue.offerAll(events, [
4175
+ childThreadCreatedEvent(ids),
4176
+ childAssistantSnapshot(ids),
4177
+ linkedSubagentEvent(ids, "running"),
4178
+ ]);
4179
+ yield* Deferred.await(subagentIngested);
4180
+});
4181
+
4182
+it.effect("settles a child message only an idle flush wrote when the root is interrupted", () =>
4183
+ Effect.gen(function* () {
4184
+ const ids = backgroundScenarioIds("idle-child-interrupt");
4185
+ const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event, Cause.Done>();
4186
+ const subagentIngested = yield* Deferred.make<void>();
4187
+ const flushed = yield* Deferred.make<void>();
4188
+ const finalized = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);
4189
+ const scenario = yield* runBackgroundItemScenario(
4190
+ "idle-child-interrupt",
4191
+ () => Stream.fromQueue(events),
4192
+ {
4193
+ onIngest: (event) =>
4194
+ event.type === "subagent.updated"
4195
+ ? Deferred.succeed(subagentIngested, undefined)
4196
+ : isHeldChildSnapshot(event)
4197
+ ? Deferred.succeed(flushed, undefined)
4198
+ : Effect.void,
4199
+ onFinalized: (written) => Ref.set(finalized, written),
4200
+ },
4201
+ ).pipe(Effect.forkChild);
4202
+
4203
+ yield* holdChildSnapshot(ids, events, subagentIngested);
4204
+ yield* TestClock.adjust("1 second");
4205
+ yield* Deferred.await(flushed);
4206
+ yield* Queue.offer(events, rootTerminalEvent(ids, "interrupted"));
4207
+ yield* Fiber.join(scenario);
4208
+
4209
+ const settled = (yield* Ref.get(finalized)).flatMap((event) =>
4210
+ event.type === "turn-item.updated" && event.payload.id === ids.childItemId
4211
+ ? [event.payload]
4212
+ : [],
4213
+ );
4214
+ assert.equal(settled.length, 1);
4215
+ assert.equal(settled[0]?.status, "interrupted");
4216
+ assert.equal("text" in settled[0]! ? settled[0].text : null, heldChildText);
4217
+ assert.isFalse("streaming" in settled[0]! && settled[0].streaming);
4218
+ }),
4219
+);
4220
+
4221
+it.effect.each([
4222
+ { name: "completed", terminalStatus: "completed" as const },
4223
+ { name: "failed with its children continuing", terminalStatus: "failed" as const },
4224
+])("idle-flushes a continuing child's text after the root $name", ({ terminalStatus }) =>
4225
+ Effect.gen(function* () {
4226
+ const key = `idle-child-after-${terminalStatus}`;
4227
+ const ids = backgroundScenarioIds(key);
4228
+ const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event, Cause.Done>();
4229
+ const subagentIngested = yield* Deferred.make<void>();
4230
+ const rootFinalized = yield* Deferred.make<void>();
4231
+ const flushed = yield* Deferred.make<void>();
4232
+ const rootWrites = yield* Ref.make<ReadonlyArray<string>>([]);
4233
+ const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(events), {
4234
+ onIngest: (event) =>
4235
+ event.type === "subagent.updated"
4236
+ ? Deferred.succeed(subagentIngested, undefined)
4237
+ : isHeldChildSnapshot(event)
4238
+ ? Deferred.succeed(flushed, undefined)
4239
+ : event.type === "message.updated" && event.message.threadId === ids.threadId
4240
+ ? Ref.update(rootWrites, (current) => [...current, event.message.text])
4241
+ : Effect.void,
4242
+ onFinalized: () => Deferred.succeed(rootFinalized, undefined),
4243
+ }).pipe(Effect.forkChild);
4244
+
4245
+ yield* holdChildSnapshot(ids, events, subagentIngested);
4246
+ // The root ends mid-paragraph: its tail is held, and the run settles without it.
4247
+ yield* Queue.offer(
4248
+ events,
4249
+ streamedMessageEvent({
4250
+ id: `message:${key}:root`,
4251
+ threadId: ids.threadId,
4252
+ runId: ids.runId,
4253
+ role: "assistant",
4254
+ text: "Root.\n\nhalf",
4255
+ streaming: true,
4256
+ }),
4257
+ );
4258
+ const terminal = rootTerminalEvent(ids, terminalStatus);
4259
+ yield* Queue.offer(
4260
+ events,
4261
+ terminalStatus === "failed"
4262
+ ? ({ ...terminal, childTurnsContinue: true } as ProviderAdapter.ProviderAdapterV2Event)
4263
+ : terminal,
4264
+ );
4265
+ yield* Deferred.await(rootFinalized);
4266
+ yield* TestClock.adjust("1 second");
4267
+ yield* Deferred.await(flushed);
4268
+ yield* Queue.offer(events, linkedSubagentEvent(ids, "completed"));
4269
+ const observed = yield* Fiber.join(scenario);
4270
+
4271
+ assert.deepEqual(observed.slice(-3), [
4272
+ "root-finalized",
4273
+ "turn_item:running",
4274
+ "subagent:completed",
4275
+ ]);
4276
+ // A streaming write after the run settled would reopen the root's message.
4277
+ assert.deepEqual(yield* Ref.get(rootWrites), ["Root.\n\n"]);
4278
+ }),
4279
+);
4280
+
4281
+function streamedMessageEvent(input: {
4282
+ readonly id: string;
4283
+ readonly threadId: ThreadId;
4284
+ readonly runId: RunId | null;
4285
+ readonly role: "assistant" | "user";
4286
+ readonly text: string;
4287
+ readonly streaming: boolean;
4288
+}): ProviderAdapter.ProviderAdapterV2Event {
4289
+ const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z");
4290
+ return {
4291
+ type: "message.updated",
4292
+ driver,
4293
+ message: {
4294
+ createdBy: input.role === "user" ? "user" : "agent",
4295
+ creationSource: "provider",
4296
+ id: MessageId.make(input.id),
4297
+ threadId: input.threadId,
4298
+ runId: input.runId,
4299
+ nodeId: null,
4300
+ role: input.role,
4301
+ text: input.text,
4302
+ attachments: [],
4303
+ streaming: input.streaming,
4304
+ createdAt: now,
4305
+ updatedAt: now,
4306
+ },
4307
+ } as ProviderAdapter.ProviderAdapterV2Event;
4308
+}
4309
+
4310
+it.effect("drops a child's held text once the child is handed to another run", () =>
4311
+ Effect.gen(function* () {
4312
+ const key = "idle-child-handoff";
4313
+ const ids = backgroundScenarioIds(key);
4314
+ const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event, Cause.Done>();
4315
+ const subagentIngested = yield* Deferred.make<void>();
4316
+ const handoffRead = yield* Deferred.make<void>();
4317
+ const rootFlushed = yield* Deferred.make<void>();
4318
+ const staleWrites = yield* Ref.make<ReadonlyArray<string>>([]);
4319
+ const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(events), {
4320
+ onIngest: (event) => {
4321
+ if (event.type === "subagent.updated") return Deferred.succeed(subagentIngested, undefined);
4322
+ if (event.type !== "message.updated" && event.type !== "turn_item.updated")
4323
+ return Effect.void;
4324
+ const message = event.type === "message.updated" ? event.message : event.turnItem;
4325
+ if (message.threadId === ids.childThreadId)
4326
+ return Ref.update(staleWrites, (current) => [...current, event.type]);
4327
+ if (event.type === "message.updated" && event.message.role === "user")
4328
+ return Deferred.succeed(handoffRead, undefined);
4329
+ return Deferred.succeed(rootFlushed, undefined);
4330
+ },
4331
+ }).pipe(Effect.forkChild);
4332
+
4333
+ yield* holdChildSnapshot(ids, events, subagentIngested);
4334
+ const handoff = linkedSubagentEvent(ids, "running");
4335
+ yield* Queue.offerAll(events, [
4336
+ streamedMessageEvent({
4337
+ id: `message:${key}:child`,
4338
+ threadId: ids.childThreadId,
4339
+ runId: null,
4340
+ role: "assistant",
4341
+ text: heldChildText,
4342
+ streaming: true,
4343
+ }),
4344
+ handoff.type === "subagent.updated"
4345
+ ? { ...handoff, subagent: { ...handoff.subagent, runId: RunId.make(`run:${key}:next`) } }
4346
+ : handoff,
4347
+ // Due in the same flush as the child's text, after it.
4348
+ streamedMessageEvent({
4349
+ id: `message:${key}:root`,
4350
+ threadId: ids.threadId,
4351
+ runId: ids.runId,
4352
+ role: "assistant",
4353
+ text: "Root partial",
4354
+ streaming: true,
4355
+ }),
4356
+ streamedMessageEvent({
4357
+ id: `message:${key}:marker`,
4358
+ threadId: ids.threadId,
4359
+ runId: ids.runId,
4360
+ role: "user",
4361
+ text: "marker",
4362
+ streaming: false,
4363
+ }),
4364
+ ]);
4365
+ yield* Deferred.await(handoffRead);
4366
+ yield* TestClock.adjust("1 second");
4367
+ yield* Deferred.await(rootFlushed);
4368
+ yield* Queue.end(events);
4369
+ yield* Fiber.join(scenario);
4370
+
4371
+ assert.deepEqual(yield* Ref.get(staleWrites), []);
4372
+ }),
4373
+);
apps/server/src/orchestration-v2/RunExecutionService.ts
+147
-60
@@ -32,9 +32,11 @@ import * as Effect from "effect/Effect";
32
import * as Exit from "effect/Exit";
33
import * as Fiber from "effect/Fiber";
34
import * as Layer from "effect/Layer";
35
+import * as Queue from "effect/Queue";
36
import * as Ref from "effect/Ref";
37
import * as Schedule from "effect/Schedule";
38
import * as Schema from "effect/Schema";
39
+import * as Stream from "effect/Stream";
40
41
import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts";
42
import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts";
@@ -174,6 +176,12 @@ function isRunOwnedSubagentTerminalStatus(
176
return status === "interrupted" || status === "failed" || status === "cancelled";
177
}
178
179
+function streamedThreadId(event: ProviderAdapter.ProviderAdapterV2Event): ThreadId | null {
180
+ if (event.type === "message.updated") return event.message.threadId;
181
+ if (event.type === "turn_item.updated") return event.turnItem.threadId;
182
+ return null;
183
+}
184
+
185
function childTurnsContinue(terminal: ProviderTerminalEvent): boolean {
186
return terminal.status === "failed" && terminal.childTurnsContinue === true;
187
}
@@ -1073,6 +1081,12 @@ export const layer: Layer.Layer<
1081
ReadonlySet<OrchestrationV2TurnItem["id"]>
1082
>(new Set(inheritedBackgroundTurnItemsById.keys()));
1083
const openRunOwnedSubagents = yield* Ref.make(emptyOpenRunOwnedSubagentProjection());
1084
+ // The last stored root snapshot's pendingBackgroundWork; adapters that omit it leave it false.
1085
+ let rootSnapshotPendingWork = false;
1086
+ const assistantStreaming = makeAssistantStreamingFilter(responseStreamingMode);
1087
+ // Set when ingestion ends or finalization settles the run's open items, which a later
1088
+ // streaming snapshot must not reopen.
1089
+ let idleFlushClosed = false;
1090
const finalizeRootRun = Effect.fn("RunExecutionService.finalizeRootRun")(function* (
1091
terminal: ProviderTerminalEvent,
1092
) {
@@ -1115,6 +1129,7 @@ export const layer: Layer.Layer<
1129
);
1130
if (isRunOwnedSubagentTerminalStatus(terminal.status) && !childTurnsContinue(terminal)) {
1131
yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection());
1132
+ idleFlushClosed = true;
1133
}
1134
yield* Ref.set(rootRunFinalized, true);
1135
});
@@ -1301,6 +1316,10 @@ export const layer: Layer.Layer<
1316
if (yield* Ref.get(providerThreadOwnerLost)) {
1317
return true;
1318
}
1319
+ // The live probe reads adapter state, which can be ahead of the events consumed here.
1320
+ if (rootSnapshotPendingWork) {
1321
+ return false;
1322
+ }
1323
// Claude background Bash has no turn-item projection. Use only the
1324
// thread-scoped probe: session-wide pending work (siblings, wake
1325
// buffers, session subagents) must not pin this root subscription.
@@ -1315,7 +1334,6 @@ export const layer: Layer.Layer<
1334
}
1335
return true;
1336
});
1318
- const filterAssistantEvent = makeAssistantStreamingFilter(responseStreamingMode);
1337
// Held text writes nothing, so its item would take a position only
1338
// when finally written, behind every item that began after it.
1339
const reservedHeldTurnItemIds = new Set<TurnItemId>();
@@ -1398,6 +1416,121 @@ export const layer: Layer.Layer<
1416
}
1417
return { event, accepted };
1418
});
1419
+ // Per-event ingestion avoids adding a span for every provider snapshot.
1420
+ const ingestDelivered = Effect.fnUntraced(function* (
1421
+ event: ProviderAdapter.ProviderAdapterV2Event,
1422
+ deliveredEvent: ProviderAdapter.ProviderAdapterV2Event,
1423
+ ) {
1424
+ const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen);
1425
+ const isRootProviderThreadUpdate =
1426
+ event.type === "provider_thread.updated" &&
1427
+ event.providerThread.id === input.providerThread.id;
1428
+ const nativeChildTurn =
1429
+ event.type === "provider_turn.updated" &&
1430
+ event.providerTurn.providerThreadId !== input.providerThread.id;
1431
+ if (event.type === "provider_turn.updated" && !nativeChildTurn) {
1432
+ firstRootProviderTurnId ??= event.providerTurn.id;
1433
+ }
1434
+ const storedEvents = yield* providerEventIngestor.ingestNormalized({
1435
+ analyticsContext: nativeChildTurn
1436
+ ? {
1437
+ ...analyticsContext,
1438
+ origin: NATIVE_SUBAGENT_TURN_ORIGIN,
1439
+ segment: "native_child",
1440
+ }
1441
+ : {
1442
+ ...analyticsContext,
1443
+ segment:
1444
+ event.type === "provider_turn.updated" &&
1445
+ event.providerTurn.id !== firstRootProviderTurnId
1446
+ ? "continuation"
1447
+ : input.attempt.reason,
1448
+ },
1449
+ providerSessionId: input.providerSessionId,
1450
+ providerInstanceId: input.run.providerInstanceId,
1451
+ threadId: input.run.threadId,
1452
+ runId: input.run.id,
1453
+ nodeId: input.rootNode.id,
1454
+ event: deliveredEvent,
1455
+ ...(isRootProviderThreadUpdate
1456
+ ? rootTerminalAlreadySeen
1457
+ ? {
1458
+ writeIfProviderThreadOwner: {
1459
+ providerThreadId: input.providerThread.id,
1460
+ runId: input.run.id,
1461
+ activeAttemptId: input.attempt.id,
1462
+ expectedLastRunOrdinal: input.run.ordinal,
1463
+ },
1464
+ }
1465
+ : {
1466
+ writeIfRunCurrent: {
1467
+ runId: input.run.id,
1468
+ activeAttemptId: input.attempt.id,
1469
+ expectedStatus: "running" as const,
1470
+ },
1471
+ }
1472
+ : {}),
1473
+ });
1474
+ if (isRootProviderThreadUpdate && rootTerminalAlreadySeen && storedEvents.length === 0) {
1475
+ yield* Ref.set(providerThreadOwnerLost, true);
1476
+ }
1477
+ return storedEvents.length;
1478
+ });
1479
+ // The timer only wakes ingestion. The flush runs as an ingestion step outside any provider
1480
+ // batch, so it stays ordered with routing and snapshots, and a write has committed when
1481
+ // it returns.
1482
+ const idleFlushTicks = yield* Queue.dropping<{
1483
+ readonly type: "idle_flush";
1484
+ readonly at: number;
1485
+ }>(1);
1486
+ let idleFlushWake: { readonly at: number; readonly fiber: Fiber.Fiber<void> } | undefined;
1487
+ const scheduleIdleFlush = Effect.suspend(() => {
1488
+ const at = assistantStreaming.nextIdleFlushAt();
1489
+ if (
1490
+ idleFlushClosed ||
1491
+ at === null ||
1492
+ (idleFlushWake !== undefined && idleFlushWake.at <= at)
1493
+ )
1494
+ return Effect.void;
1495
+ const previous = idleFlushWake?.fiber;
1496
+ return Effect.gen(function* () {
1497
+ if (previous !== undefined) yield* Fiber.interrupt(previous);
1498
+ const now = DateTime.toEpochMillis(yield* DateTime.now);
1499
+ const fiber = yield* Effect.sleep(Duration.millis(Math.max(0, at - now))).pipe(
1500
+ Effect.andThen(Queue.offer(idleFlushTicks, { type: "idle_flush", at })),
1501
+ Effect.asVoid,
1502
+ Effect.forkDetach,
1503
+ );
1504
+ idleFlushWake = { at, fiber };
1505
+ });
1506
+ });
1507
+ const closeIdleFlush = Effect.suspend(() => {
1508
+ idleFlushClosed = true;
1509
+ const wake = idleFlushWake;
1510
+ idleFlushWake = undefined;
1511
+ return wake === undefined ? Effect.void : Fiber.interrupt(wake.fiber);
1512
+ });
1513
+ // Per-tick ingestion avoids adding a span for every idle wake.
1514
+ const flushIdleAssistantText = Effect.fnUntraced(function* (wokeAt: number) {
1515
+ // A tick means its timer slept until `wokeAt`, but the wall clock can still read up
1516
+ // to a millisecond earlier; trusting it would keep the finished wake and never re-arm.
1517
+ const now = Math.max(DateTime.toEpochMillis(yield* DateTime.now), wokeAt);
1518
+ // A wake due by now has fired; a later deadline needs a new one.
1519
+ if (idleFlushWake !== undefined && idleFlushWake.at <= now) idleFlushWake = undefined;
1520
+ if (idleFlushClosed) return;
1521
+ const rootSettled = yield* Ref.get(rootRunFinalized);
1522
+ for (const event of assistantStreaming.takeIdleFlushes(now)) {
1523
+ // Settling the root run closed its messages; only children may still stream.
1524
+ if (rootSettled && streamedThreadId(event) === input.run.threadId) continue;
1525
+ // The held text's child may have been handed to another run since it arrived.
1526
+ if (!routeProviderEvent(event, routeIdentity, yield* Ref.get(eventRouting))[0])
1527
+ continue;
1528
+ // A failed write fails ingestion like any provider write.
1529
+ yield* ingestDelivered(event, event);
1530
+ yield* trackChildLifecycle(event, true);
1531
+ }
1532
+ yield* scheduleIdleFlush;
1533
+ });
1534
const ingestEvent = Effect.fnUntraced(function* ({
1535
event,
1536
accepted,
@@ -1414,65 +1547,12 @@ export const layer: Layer.Layer<
1547
yield* Ref.set(latestRootProviderTurn, event.providerTurn);
1548
}
1549
let storedEventCount = 0;
1417
- const deliveredEvent = filterAssistantEvent(
1550
+ const deliveredEvent = assistantStreaming.filter(
1551
event,
1552
DateTime.toEpochMillis(yield* DateTime.now),
1553
);
1554
if (deliveredEvent) {
1422
- const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen);
1423
- const isRootProviderThreadUpdate =
1424
- event.type === "provider_thread.updated" &&
1425
- event.providerThread.id === input.providerThread.id;
1426
- const nativeChildTurn =
1427
- event.type === "provider_turn.updated" &&
1428
- event.providerTurn.providerThreadId !== input.providerThread.id;
1429
- if (event.type === "provider_turn.updated" && !nativeChildTurn) {
1430
- firstRootProviderTurnId ??= event.providerTurn.id;
1431
- }
1432
- const storedEvents = yield* providerEventIngestor.ingestNormalized({
1433
- analyticsContext: nativeChildTurn
1434
- ? {
1435
- ...analyticsContext,
1436
- origin: NATIVE_SUBAGENT_TURN_ORIGIN,
1437
- segment: "native_child",
1438
- }
1439
- : {
1440
- ...analyticsContext,
1441
- segment:
1442
- event.type === "provider_turn.updated" &&
1443
- event.providerTurn.id !== firstRootProviderTurnId
1444
- ? "continuation"
1445
- : input.attempt.reason,
1446
- },
1447
- providerSessionId: input.providerSessionId,
1448
- providerInstanceId: input.run.providerInstanceId,
1449
- threadId: input.run.threadId,
1450
- runId: input.run.id,
1451
- nodeId: input.rootNode.id,
1452
- event: deliveredEvent,
1453
- ...(isRootProviderThreadUpdate
1454
- ? rootTerminalAlreadySeen
1455
- ? {
1456
- writeIfProviderThreadOwner: {
1457
- providerThreadId: input.providerThread.id,
1458
- runId: input.run.id,
1459
- activeAttemptId: input.attempt.id,
1460
- expectedLastRunOrdinal: input.run.ordinal,
1461
- },
1462
- }
1463
- : {
1464
- writeIfRunCurrent: {
1465
- runId: input.run.id,
1466
- activeAttemptId: input.attempt.id,
1467
- expectedStatus: "running" as const,
1468
- },
1469
- }
1470
- : {}),
1471
- });
1472
- storedEventCount = storedEvents.length;
1473
- if (isRootProviderThreadUpdate && rootTerminalAlreadySeen && storedEventCount === 0) {
1474
- yield* Ref.set(providerThreadOwnerLost, true);
1475
- }
1555
+ storedEventCount = yield* ingestDelivered(event, deliveredEvent);
1556
} else if (
1557
event.type === "turn_item.updated" &&
1558
(event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") &&
@@ -1489,6 +1569,7 @@ export const layer: Layer.Layer<
1569
if (event.type === "provider_thread.updated") {
1570
if (event.providerThread.id === input.providerThread.id && storedEventCount > 0) {
1571
yield* Ref.set(latestProviderThread, event.providerThread);
1572
+ rootSnapshotPendingWork = event.pendingBackgroundWork === true;
1573
}
1574
}
1575
if (
@@ -1512,14 +1593,19 @@ export const layer: Layer.Layer<
1593
}
1594
}
1595
yield* trackChildLifecycle(event, deliveredEvent !== null);
1596
+ yield* scheduleIdleFlush;
1597
});
1598
const providerEventFiber = yield* eventSink
1599
.consumeProviderEvents(
1518
- eventSubscription.events,
1600
+ eventSubscription.events.pipe(
1601
+ Stream.merge(Stream.fromQueue(idleFlushTicks), { haltStrategy: "left" }),
1602
+ ),
1603
(event) =>
1520
- Effect.gen(function* () {
1521
- yield* ingestEvent(yield* routeEvent(event));
1522
- }),
1604
+ event.type === "idle_flush"
1605
+ ? flushIdleAssistantText(event.at)
1606
+ : Effect.gen(function* () {
1607
+ yield* ingestEvent(yield* routeEvent(event));
1608
+ }),
1609
Effect.gen(function* () {
1610
if (yield* agentTurnsSettled) yield* endPreviewActivity;
1611
return yield* shouldStopProviderEventIngestion;
@@ -1527,6 +1613,7 @@ export const layer: Layer.Layer<
1613
Ref.get(rootTerminalSeen).pipe(Effect.map((seen) => !seen)),
1614
)
1615
.pipe(
1616
+ Effect.ensuring(closeIdleFlush),
1617
Effect.mapError((cause) => new RunExecutionIngestError({ runId: input.run.id, cause })),
1618
Effect.flatMap(() =>
1619
Effect.gen(function* () {
apps/server/src/orchestration-v2/assistantStreaming.test.ts
+171
-2
@@ -1,5 +1,8 @@
1
-import { describe, expect, it } from "vite-plus/test";
2
-import { splitBufferedAssistantText } from "./assistantStreaming.ts";
1
+import { MessageId, ProviderDriverKind, ThreadId } from "@t3tools/contracts";
2
+import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
3
+import * as DateTime from "effect/DateTime";
4
+import { assert, describe, expect, it } from "vite-plus/test";
5
+import { makeAssistantStreamingFilter, splitBufferedAssistantText } from "./assistantStreaming.ts";
6
7
describe("V2 assistant streaming", () => {
8
it("keeps code fences intact", () => {
@@ -24,3 +27,169 @@ it("holds a streamed section heading until its content has a boundary", () => {
27
rest: "Next",
28
});
29
});
30
+
31
+type Step =
32
+ // A snapshot at `at` ms, and the text it delivers at once (null when held back).
33
+ | { readonly at: number; readonly text: string; readonly shows: string | null }
34
+ | { readonly at: number; readonly final: string }
35
+ | { readonly id: string; readonly at: number; readonly text: string; readonly shows: string }
36
+ // Held snapshots due by `idle` ms, delivered whole.
37
+ | { readonly idle: number; readonly shows: ReadonlyArray<string> }
38
+ | { readonly dueAt: number | null };
39
+
40
+const createdAt = DateTime.makeUnsafe(0);
41
+const snapshot = (id: string, text: string, streaming: boolean) =>
42
+ ({
43
+ type: "message.updated",
44
+ driver: ProviderDriverKind.make("codex"),
45
+ message: {
46
+ createdBy: "agent",
47
+ creationSource: "provider",
48
+ id: MessageId.make(id),
49
+ threadId: ThreadId.make("thread"),
50
+ runId: null,
51
+ nodeId: null,
52
+ role: "assistant",
53
+ text,
54
+ attachments: [],
55
+ streaming,
56
+ createdAt,
57
+ updatedAt: createdAt,
58
+ },
59
+ }) satisfies ProviderAdapter.ProviderAdapterV2Event;
60
+const shown = (event: ProviderAdapter.ProviderAdapterV2Event | null) =>
61
+ event?.type === "message.updated" ? event.message.text : null;
62
+
63
+it.each<{ readonly name: string; readonly mode?: "turn"; readonly steps: ReadonlyArray<Step> }>([
64
+ {
65
+ name: "an unfinished code fence",
66
+ steps: [
67
+ { at: 0, text: "A\n\n```ts\nx()", shows: "A\n\n" },
68
+ { dueAt: 800 },
69
+ { idle: 799, shows: [] },
70
+ { idle: 800, shows: ["A\n\n```ts\nx()"] },
71
+ { dueAt: null },
72
+ { at: 1_300, text: "A\n\n```ts\nx()\n```\n\nB", shows: "A\n\n```ts\nx()\n```\n\n" },
73
+ ],
74
+ },
75
+ {
76
+ name: "a list's partial last item",
77
+ steps: [
78
+ { at: 0, text: "Steps:\n\n- one\n- tw", shows: "Steps:\n\n- one\n" },
79
+ { idle: 800, shows: ["Steps:\n\n- one\n- tw"] },
80
+ { at: 1_300, text: "Steps:\n\n- one\n- two\n- th", shows: "Steps:\n\n- one\n- two\n" },
81
+ ],
82
+ },
83
+ {
84
+ name: "an incomplete table row",
85
+ steps: [
86
+ { at: 0, text: "Intro.\n\n| a | b |\n|---|---|\n| 1 |", shows: "Intro.\n\n" },
87
+ { idle: 800, shows: ["Intro.\n\n| a | b |\n|---|---|\n| 1 |"] },
88
+ {
89
+ at: 1_300,
90
+ text: "Intro.\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\nAfter",
91
+ shows: "Intro.\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\n",
92
+ },
93
+ ],
94
+ },
95
+ {
96
+ name: "more of the same paragraph after an idle delivery",
97
+ steps: [
98
+ { at: 0, text: "A\n\nB", shows: "A\n\n" },
99
+ { idle: 800, shows: ["A\n\nB"] },
100
+ { at: 900, text: "A\n\nB more", shows: null },
101
+ { idle: 1_699, shows: [] },
102
+ { idle: 1_700, shows: ["A\n\nB more"] },
103
+ { idle: 5_000, shows: [] },
104
+ // Its only boundary is behind what the idle delivery showed.
105
+ { at: 5_100, text: "A\n\nB more and", shows: null },
106
+ { at: 5_200, text: "A\n\nB more and.\n\nC", shows: "A\n\nB more and.\n\n" },
107
+ { at: 5_300, final: "A\n\nB more and.\n\nC." },
108
+ { idle: 10_000, shows: [] },
109
+ ],
110
+ },
111
+ {
112
+ name: "completion at the idle deadline",
113
+ steps: [
114
+ { at: 0, text: "A\n\nB", shows: "A\n\n" },
115
+ { at: 800, final: "A\n\nB." },
116
+ { idle: 800, shows: [] },
117
+ ],
118
+ },
119
+ {
120
+ name: "completion right after an idle delivery at the same instant",
121
+ steps: [
122
+ { at: 0, text: "A\n\nB", shows: "A\n\n" },
123
+ { idle: 800, shows: ["A\n\nB"] },
124
+ { at: 800, final: "A\n\nB." },
125
+ { idle: 5_000, shows: [] },
126
+ ],
127
+ },
128
+ {
129
+ name: "several fast paragraphs inside the throttle",
130
+ steps: [
131
+ { at: 0, text: "A\n\nB", shows: "A\n\n" },
132
+ { at: 100, text: "A\n\nB\n\nC", shows: null },
133
+ { at: 200, text: "A\n\nB\n\nC\n\nD", shows: null },
134
+ { at: 300, text: "A\n\nB\n\nC\n\nD\n\nE", shows: null },
135
+ { idle: 1_099, shows: [] },
136
+ { idle: 1_100, shows: ["A\n\nB\n\nC\n\nD\n\nE"] },
137
+ ],
138
+ },
139
+ {
140
+ name: "two messages due at different times",
141
+ steps: [
142
+ { id: "a", at: 0, text: "A\n\nx", shows: "A\n\n" },
143
+ { id: "b", at: 300, text: "B\n\ny", shows: "B\n\n" },
144
+ { dueAt: 800 },
145
+ { idle: 800, shows: ["A\n\nx"] },
146
+ { dueAt: 1_100 },
147
+ { idle: 1_100, shows: ["B\n\ny"] },
148
+ ],
149
+ },
150
+ {
151
+ name: "empty text",
152
+ steps: [{ at: 0, text: "", shows: null }, { dueAt: null }],
153
+ },
154
+ {
155
+ name: "whitespace-only text",
156
+ steps: [
157
+ { at: 0, text: " \n", shows: null },
158
+ { idle: 800, shows: [" \n"] },
159
+ { idle: 5_000, shows: [] },
160
+ ],
161
+ },
162
+ {
163
+ // Rewrites replace delivered text: shorter at once, same length after the throttle.
164
+ name: "provider rewrites and truncation",
165
+ steps: [
166
+ { at: 0, text: "Original long paragraph.\n\n", shows: "Original long paragraph.\n\n" },
167
+ { at: 1_000, text: "New.\n\n", shows: "New.\n\n" },
168
+ { at: 1_100, text: "Old.\n\n", shows: null },
169
+ { idle: 1_900, shows: ["Old.\n\n"] },
170
+ { at: 2_500, text: "Old", shows: null },
171
+ { idle: 3_300, shows: ["Old"] },
172
+ { at: 4_000, text: "", shows: null },
173
+ { idle: 4_800, shows: [""] },
174
+ ],
175
+ },
176
+ {
177
+ name: "turn mode",
178
+ mode: "turn",
179
+ steps: [{ at: 0, text: "A\n\nB", shows: null }, { dueAt: null }],
180
+ },
181
+])("paragraph streaming holds and idle-delivers $name", ({ mode, steps }) => {
182
+ const filter = makeAssistantStreamingFilter(mode ?? "paragraph", 800);
183
+ for (const step of steps) {
184
+ if ("dueAt" in step) assert.equal(filter.nextIdleFlushAt(), step.dueAt);
185
+ else if ("idle" in step)
186
+ assert.deepEqual(filter.takeIdleFlushes(step.idle).map(shown), [...step.shows]);
187
+ else if ("final" in step)
188
+ assert.equal(shown(filter.filter(snapshot("m", step.final, false), step.at)), step.final);
189
+ else
190
+ assert.equal(
191
+ shown(filter.filter(snapshot("id" in step ? step.id : "m", step.text, true), step.at)),
192
+ step.shows,
193
+ );
194
+ }
195
+});
apps/server/src/orchestration-v2/assistantStreaming.ts
+124
-39
@@ -68,44 +68,129 @@ export function splitBufferedAssistantText(text: string): { ready: string; rest:
68
return { ready: text.slice(0, boundary), rest: text.slice(boundary) };
69
}
70
71
-export function makeAssistantStreamingFilter(mode: ResponseStreamingMode) {
72
- const delivered = new Map<string, { text: string; at: number }>();
73
- return (
74
- event: ProviderAdapter.ProviderAdapterV2Event,
75
- now: number,
76
- ): ProviderAdapter.ProviderAdapterV2Event | null => {
77
- if (
78
- event.type === "node.updated" &&
79
- event.node.kind === "assistant_message" &&
80
- event.node.status === "running"
81
- )
82
- return null;
83
- const message =
84
- event.type === "message.updated" && event.message.role === "assistant"
85
- ? event.message
86
- : event.type === "turn_item.updated" &&
87
- (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning")
88
- ? event.turnItem
89
- : null;
90
- if (!message) return event;
91
- const key = `${event.type}:${message.id}`;
92
- if (!message.streaming) {
93
- delivered.delete(key);
94
- return event;
95
- }
96
- if (mode === "turn") return null;
97
- const previous = delivered.get(key);
98
- if (previous && now - previous.at < 400) return null;
99
- const { ready } = splitBufferedAssistantText(message.text);
100
- if (!ready || ready === previous?.text) return null;
101
- delivered.set(key, { text: ready, at: now });
102
- if (event.type === "message.updated")
103
- return { ...event, message: { ...event.message, text: ready } };
104
- if (
105
- event.type === "turn_item.updated" &&
106
- (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning")
107
- )
108
- return { ...event, turnItem: { ...event.turnItem, text: ready } };
109
- return event;
71
+// Long enough to ride out normal gaps between chunks; short enough that text written before a
72
+// long tool call or background wait does not look lost.
73
+const ASSISTANT_IDLE_FLUSH_MS = 800;
74
+
75
+type StreamedMessage = {
76
+ readonly id: string;
77
+ readonly text: string;
78
+ readonly streaming: boolean;
79
+};
80
+
81
+interface StreamTrack {
82
+ readonly text: string;
83
+ readonly at: number;
84
+ // Latest snapshot whose text has not all been delivered yet.
85
+ readonly held: ProviderAdapter.ProviderAdapterV2Event | null;
86
+ readonly dueAt: number;
87
+}
88
+
89
+function streamedMessage(event: ProviderAdapter.ProviderAdapterV2Event): StreamedMessage | null {
90
+ if (event.type === "message.updated" && event.message.role === "assistant") return event.message;
91
+ if (
92
+ event.type === "turn_item.updated" &&
93
+ (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning")
94
+ )
95
+ return event.turnItem;
96
+ return null;
97
+}
98
+
99
+function withText(
100
+ event: ProviderAdapter.ProviderAdapterV2Event,
101
+ text: string,
102
+): ProviderAdapter.ProviderAdapterV2Event {
103
+ if (event.type === "message.updated") return { ...event, message: { ...event.message, text } };
104
+ if (
105
+ event.type === "turn_item.updated" &&
106
+ (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning")
107
+ )
108
+ return { ...event, turnItem: { ...event.turnItem, text } };
109
+ return event;
110
+}
111
+
112
+/**
113
+ * Paragraph mode delivers streamed text at Markdown boundaries, at most every 400ms. A snapshot
114
+ * held back that way is delivered whole once its message has been idle for `idleFlushMs`, so a
115
+ * provider that pauses mid-turn still shows everything it wrote.
116
+ */
117
+export function makeAssistantStreamingFilter(
118
+ mode: ResponseStreamingMode,
119
+ idleFlushMs = ASSISTANT_IDLE_FLUSH_MS,
120
+) {
121
+ const tracks = new Map<string, StreamTrack>();
122
+ // Earliest deadline of any held snapshot, or earlier: a newer snapshot only pushes its own
123
+ // deadline later, so the cache stays O(1) per event and is recomputed when it comes due.
124
+ let nextDueAt: number | null = null;
125
+ const track = (key: string, value: StreamTrack) => {
126
+ tracks.set(key, value);
127
+ if (value.held !== null && (nextDueAt === null || value.dueAt < nextDueAt))
128
+ nextDueAt = value.dueAt;
129
+ };
130
+ return {
131
+ filter(
132
+ event: ProviderAdapter.ProviderAdapterV2Event,
133
+ now: number,
134
+ ): ProviderAdapter.ProviderAdapterV2Event | null {
135
+ if (
136
+ event.type === "node.updated" &&
137
+ event.node.kind === "assistant_message" &&
138
+ event.node.status === "running"
139
+ )
140
+ return null;
141
+ const message = streamedMessage(event);
142
+ if (!message) return event;
143
+ const key = `${event.type}:${message.id}`;
144
+ if (!message.streaming) {
145
+ tracks.delete(key);
146
+ return event;
147
+ }
148
+ if (mode === "turn") return null;
149
+ const previous = tracks.get(key);
150
+ const delivered = previous?.text ?? "";
151
+ const hold = () => {
152
+ track(key, {
153
+ text: delivered,
154
+ at: previous?.at ?? Number.NEGATIVE_INFINITY,
155
+ held: message.text !== delivered ? event : null,
156
+ dueAt: now + idleFlushMs,
157
+ });
158
+ return null;
159
+ };
160
+ if (previous && now - previous.at < 400) return hold();
161
+ const { ready } = splitBufferedAssistantText(message.text);
162
+ // An idle delivery may already show more than the next boundary; never shrink it. A
163
+ // provider rewrite or truncation is not a prefix and replaces it.
164
+ if (delivered.startsWith(ready)) return hold();
165
+ track(key, {
166
+ text: ready,
167
+ at: now,
168
+ held: ready !== message.text ? event : null,
169
+ dueAt: now + idleFlushMs,
170
+ });
171
+ return withText(event, ready);
172
+ },
173
+ nextIdleFlushAt: (): number | null => nextDueAt,
174
+ /** Returns the held snapshots due by `now` and counts them as delivered. */
175
+ takeIdleFlushes(now: number): ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event> {
176
+ const due: Array<ProviderAdapter.ProviderAdapterV2Event> = [];
177
+ nextDueAt = null;
178
+ for (const [key, current] of tracks) {
179
+ const event = current.held;
180
+ if (event === null) continue;
181
+ if (current.dueAt > now) {
182
+ if (nextDueAt === null || current.dueAt < nextDueAt) nextDueAt = current.dueAt;
183
+ continue;
184
+ }
185
+ due.push(event);
186
+ tracks.set(key, {
187
+ text: streamedMessage(event)?.text ?? current.text,
188
+ at: now,
189
+ held: null,
190
+ dueAt: now,
191
+ });
192
+ }
193
+ return due;
194
+ },
195
};
196
}