@devries-cameron20 / t3code Mirror · read-only / commits / e89abadaee
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 00:45 UTC

fix(server): show idle paragraphs before the assistant turn ends (#28221)

Held assistant paragraphs now appear after 800 ms without new text while the turn remains open. Idle writes preserve ingestion order, routing and finalization. Includes lean edge-case coverage and regressions for finalized-root reopening and early timer wakes. Verified with focused tests, check:changed, real web and compiled macOS captures, independent Codex review, and a fresh Claude Opus 5.5 READY review. Natural Antigravity hangs remain unverified. Implemented with Claude Opus 5.5 through Claude Code; investigated and verified with GPT-6.1-Sol through Codex in T3 Code.

shivam committed Nov 10, 2026 at 10:05 UTC e89abadaee70e0d73fb19a5ce40a686b1a92310e
6 files changed +929 -119
apps/server/src/orchestration-v2/EventSink.ts
+4 -4
@@ -12,7 +12,6 @@ import {
12 type ProjectId,
13 ThreadId,
14 } from "@t3tools/contracts";
15 -import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
15 import * as ProviderCommitBatch from "./ProviderCommitBatch.ts";
16 import * as Context from "effect/Context";
17 import * as DateTime from "effect/DateTime";
@@ -79,9 +78,10 @@ const CurrentProviderBatch = Context.Reference<ProviderBatchPublication | undefi
78 );
79
80 export interface EventSinkV2Shape {
82 - readonly consumeProviderEvents: <E, R, E2, R2>(
83 - source: Stream.Stream<ProviderAdapter.ProviderAdapterV2Event, E, R>,
84 - process: (event: ProviderAdapter.ProviderAdapterV2Event) => Effect.Effect<void, E2, R2>,
81 + /** Items that are not provider snapshots, such as a consumer's own control items, never batch. */
82 + readonly consumeProviderEvents: <A extends { readonly type: string }, E, R, E2, R2>(
83 + source: Stream.Stream<A, E, R>,
84 + process: (event: A) => Effect.Effect<void, E2, R2>,
85 after: Effect.Effect<boolean, E2, R2>,
86 canBatch: Effect.Effect<boolean, never, R2>,
87 ) => Effect.Effect<
apps/server/src/orchestration-v2/ProviderCommitBatch.ts
+4 -6
@@ -1,4 +1,3 @@
1 -import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
1 import * as Cause from "effect/Cause";
2 import * as Clock from "effect/Clock";
3 import * as Config from "effect/Config";
@@ -19,8 +18,7 @@ export const enabled = Config.Boolean("T3CODE_PROVIDER_COMMIT_BATCHING").pipe(
18 Effect.orDie,
19 );
20
22 -type Event = ProviderAdapter.ProviderAdapterV2Event;
23 -const isSnapshot = (event: Event) =>
21 +const isSnapshot = (event: { readonly type: string }) =>
22 event.type === "node.updated" ||
23 event.type === "message.updated" ||
24 event.type === "turn_item.updated" ||
@@ -232,9 +230,9 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
230 ),
231 );
232 });
235 - const consume = Effect.fnUntraced(function* <E, R, E2, R2>(
236 - source: Stream.Stream<Event, E, R>,
237 - process: (event: Event) => Effect.Effect<void, E2, R2>,
233 + const consume = Effect.fnUntraced(function* <A extends { readonly type: string }, E, R, E2, R2>(
234 + source: Stream.Stream<A, E, R>,
235 + process: (event: A) => Effect.Effect<void, E2, R2>,
236 after: Effect.Effect<boolean, E2, R2>,
237 canBatch: Effect.Effect<boolean, never, R2> = Effect.succeed(true),
238 ) {
apps/server/src/orchestration-v2/RunExecutionService.test.ts
+479 -8
@@ -14,6 +14,7 @@ import {
14 type OrchestrationV2ProviderTurn,
15 type OrchestrationV2Run,
16 type OrchestrationV2RunAttempt,
17 + type OrchestrationV2StoredEvent,
18 type OrchestrationV2Subagent,
19 type OrchestrationV2TurnItem,
20 ProviderDriverKind,
@@ -28,14 +29,19 @@ import {
29 TurnItemId,
30 } from "@t3tools/contracts";
31 import * as Cause from "effect/Cause";
32 +import * as Clock from "effect/Clock";
33 import * as Exit from "effect/Exit";
34 +import * as Fiber from "effect/Fiber";
35 import * as DateTime from "effect/DateTime";
36 import * as Deferred from "effect/Deferred";
37 +import * as Duration from "effect/Duration";
38 import * as Effect from "effect/Effect";
39 import * as Layer from "effect/Layer";
40 import * as Option from "effect/Option";
41 +import * as Queue from "effect/Queue";
42 import * as Ref from "effect/Ref";
43 import * as Stream from "effect/Stream";
44 +import * as TestClock from "effect/testing/TestClock";
45 import { CodexAppServerRequestError } from "effect-codex-app-server/errors";
46 import { SqlError, classifySqliteError } from "effect/sql/SqlError";
47
@@ -47,6 +53,7 @@ import * as EventSink from "./EventSink.ts";
53 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
54 import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
55 import * as ProjectionStore from "./ProjectionStore.ts";
56 +import * as ProviderCommitBatch from "./ProviderCommitBatch.ts";
57 import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
58 import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts";
59 import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts";
@@ -1885,6 +1892,40 @@ it.effect("keeps ingesting until queued background work clears after the probe g
1892 }),
1893 );
1894
1895 +it.effect("keeps ingesting while the last stored root snapshot reports pending work", () =>
1896 + Effect.gen(function* () {
1897 + // The live probe is already idle, as when it reads adapter state ahead of the queued events.
1898 + const consumed = yield* Deferred.make<void>();
1899 + const result = yield* Effect.raceFirst(
1900 + runBackgroundItemScenario(
1901 + "root-snapshot-pending-work",
1902 + (ids) => {
1903 + const snapshot = (pendingBackgroundWork: boolean) =>
1904 + ({
1905 + type: "provider_thread.updated",
1906 + driver,
1907 + providerThread: { id: ids.providerThreadId, appThreadId: ids.threadId },
1908 + pendingBackgroundWork,
1909 + }) as ProviderAdapter.ProviderAdapterV2Event;
1910 + return [
1911 + snapshot(true),
1912 + rootTerminalEvent(ids, "completed"),
1913 + backgroundTurnItemEvent(ids, "dynamic_tool", "completed", 1),
1914 + snapshot(false),
1915 + ];
1916 + },
1917 + {
1918 + keepEventStreamOpen: true,
1919 + hasPendingBackgroundWorkForThread: () => Effect.succeed(false),
1920 + onEventsConsumed: Deferred.succeed(consumed, undefined).pipe(Effect.asVoid),
1921 + },
1922 + ).pipe(Effect.asSome),
1923 + Deferred.await(consumed).pipe(Effect.as(Option.none())),
1924 + );
1925 + assert.deepEqual(result, Option.some(["root-finalized", "turn_item:completed"]));
1926 + }),
1927 +);
1928 +
1929 it.effect("keeps ingesting a child thread's late background item completion", () =>
1930 Effect.gen(function* () {
1931 const observed = yield* runBackgroundItemScenario("bg-child-item", (ids) => [
@@ -3692,8 +3733,18 @@ function rootTerminalEvent(
3733
3734 function runBackgroundItemScenario(
3735 key: string,
3695 - makeEvents: (ids: BackgroundScenarioIds) => ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>,
3736 + makeEvents: (
3737 + ids: BackgroundScenarioIds,
3738 + ) =>
3739 + | ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>
3740 + | Stream.Stream<ProviderAdapter.ProviderAdapterV2Event>,
3741 options?: {
3742 + readonly onIngest?: (
3743 + event: ProviderAdapter.ProviderAdapterV2Event,
3744 + write: { readonly inSharedCommit: boolean },
3745 + ) => Effect.Effect<void>;
3746 + // Consumes through the real commit coordinator, as the event sink does.
3747 + readonly batched?: boolean;
3748 readonly keepEventStreamOpen?: boolean;
3749 readonly hasPendingBackgroundWorkForThread?: (
3750 providerThread: OrchestrationV2ProviderThread,
@@ -3719,6 +3770,24 @@ function runBackgroundItemScenario(
3770 const previewActivity = yield* PreviewAgentActivity.PreviewAgentActivity.pipe(
3771 Effect.provide(PreviewAgentActivity.layer),
3772 );
3773 + let inSharedCommit = false;
3774 + const coordinator =
3775 + options?.batched === true
3776 + ? yield* ProviderCommitBatch.make(
3777 + (work) =>
3778 + Effect.suspend(() => {
3779 + inSharedCommit = true;
3780 + return work;
3781 + }).pipe(
3782 + Effect.ensuring(
3783 + Effect.sync(() => {
3784 + inSharedCommit = false;
3785 + }),
3786 + ),
3787 + ),
3788 + true,
3789 + )
3790 + : undefined;
3791 const layerTest = RunExecutionService.layer.pipe(
3792 Layer.provide(
3793 Layer.mergeAll(
@@ -3727,6 +3796,7 @@ function runBackgroundItemScenario(
3796 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
3797 layerTurnItemPositionsTest,
3798 mockEventSink({
3799 + ...(coordinator === undefined ? {} : { consumeProviderEvents: coordinator.consume }),
3800 write: () => Effect.succeed([]),
3801 writeWithEffects: (input) =>
3802 Effect.gen(function* () {
@@ -3744,9 +3814,10 @@ function runBackgroundItemScenario(
3814 }),
3815 IdAllocator.layer,
3816 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
3747 - ingestNormalized: (input) =>
3748 - Effect.gen(function* () {
3817 + ingestNormalized: (input) => {
3818 + const write = Effect.gen(function* () {
3819 const event = input.event;
3820 + yield* options?.onIngest?.(event, { inSharedCommit }) ?? Effect.void;
3821 if (event.type === "turn_item.updated") {
3822 yield* Ref.update(observed, (current) => [
3823 ...current,
@@ -3759,8 +3830,14 @@ function runBackgroundItemScenario(
3830 `subagent:${event.subagent.status}`,
3831 ]);
3832 }
3762 - return [];
3763 - }),
3833 + // Only the count of stored events matters to the run.
3834 + return event.type === "provider_thread.updated"
3835 + ? [{} as OrchestrationV2StoredEvent]
3836 + : [];
3837 + });
3838 + // A write outside a batch waits for queued snapshots, as the event sink's does.
3839 + return coordinator === undefined ? write : coordinator.barrier(write);
3840 + },
3841 }),
3842 ServerSettings.layerTest(),
3843 ),
@@ -3780,7 +3857,8 @@ function runBackgroundItemScenario(
3857 : { hasPendingBackgroundWorkForThread: options.hasPendingBackgroundWorkForThread }),
3858 subscribeEvents: Effect.gen(function* () {
3859 yield* options?.onSubscribe ?? Effect.void;
3783 - const events = Stream.fromIterable(makeEvents(ids)).pipe(
3860 + const source = makeEvents(ids);
3861 + const events = (Stream.isStream(source) ? source : Stream.fromIterable(source)).pipe(
3862 Stream.concat(
3863 Stream.fromEffect(
3864 Effect.andThen(
@@ -3847,8 +3925,9 @@ function runBackgroundItemScenario(
3925 });
3926 }).pipe(Effect.provide(layerTest));
3927
3850 - const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds"));
3851 - assert.isTrue(Option.isSome(closed), "event ingestion fiber did not finish");
3928 + // No timeout: tests advance the TestClock, which would fire it early.
3929 + yield* Deferred.await(ingestionDone);
3930 +
3931 return yield* Ref.get(observed);
3932 });
3933 }
@@ -3900,3 +3979,395 @@ it.effect("keeps a failed root's still-running native children live while the ru
3979 );
3980 }),
3981 );
3982 +
3983 +it.effect.each([
3984 + { name: "on time", wakesEarly: false },
3985 + // Timers run on the monotonic clock, so one can fire while the wall clock still reads a
3986 + // millisecond before its deadline.
3987 + { name: "before the wall clock reaches the deadline", wakesEarly: true },
3988 +])("delivers held text while an open-prompt provider idles, waking $name", ({ wakesEarly }) =>
3989 + Effect.gen(function* () {
3990 + const key = `idle-paragraph-${wakesEarly}`;
3991 + const ids = backgroundScenarioIds(key);
3992 + const now = yield* DateTime.now;
3993 + const providerEvents = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
3994 + const delivered = yield* Ref.make<ReadonlyArray<string>>([]);
3995 + const paragraphDelivered = yield* Deferred.make<void>();
3996 + const nextParagraphDelivered = yield* Deferred.make<void>();
3997 + const clock = yield* Clock.Clock;
3998 + const assistantText = (text: string, streaming: boolean) =>
3999 + [
4000 + {
4001 + type: "message.updated",
4002 + driver,
4003 + message: {
4004 + createdBy: "agent",
4005 + creationSource: "provider",
4006 + id: MessageId.make(`message:${key}`),
4007 + threadId: ids.threadId,
4008 + runId: ids.runId,
4009 + nodeId: ids.rootNodeId,
4010 + role: "assistant",
4011 + text,
4012 + attachments: [],
4013 + streaming,
4014 + createdAt: now,
4015 + updatedAt: now,
4016 + },
4017 + },
4018 + {
4019 + type: "turn_item.updated",
4020 + driver,
4021 + turnItem: {
4022 + ...makeLinkedChildTurnItemFixture({ ids, driver, type: "assistant_message" }),
4023 + id: ids.itemId,
4024 + threadId: ids.threadId,
4025 + runId: ids.runId,
4026 + nodeId: ids.rootNodeId,
4027 + providerTurnId: ids.rootProviderTurnId,
4028 + text,
4029 + streaming,
4030 + },
4031 + },
4032 + ] as ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>;
4033 +
4034 + const blockerStarted = yield* Deferred.make<void>();
4035 + const releaseBlocker = yield* Deferred.make<void>();
4036 + const blocker = streamedMessageEvent({
4037 + id: `message:${key}:blocker`,
4038 + threadId: ids.threadId,
4039 + runId: ids.runId,
4040 + role: "assistant",
4041 + text: "Other",
4042 + streaming: false,
4043 + });
4044 + const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(providerEvents), {
4045 + batched: true,
4046 + onIngest: (event, write) =>
4047 + event.type === "message.updated" && event.message.text === "Other"
4048 + ? Deferred.succeed(blockerStarted, undefined).pipe(
4049 + Effect.andThen(Deferred.await(releaseBlocker)),
4050 + )
4051 + : event.type === "message.updated" || event.type === "turn_item.updated"
4052 + ? Effect.gen(function* () {
4053 + const message = event.type === "message.updated" ? event.message : event.turnItem;
4054 + const text = "text" in message ? message.text : "";
4055 + const streaming = "streaming" in message && message.streaming;
4056 + yield* Ref.update(delivered, (current) => [
4057 + ...current,
4058 + `${event.type}:${streaming ? "streaming" : "final"}:${JSON.stringify(text)}${
4059 + write.inSharedCommit ? " in commit" : ""
4060 + }`,
4061 + ]);
4062 + if (event.type === "turn_item.updated") {
4063 + yield* Deferred.succeed(paragraphDelivered, undefined);
4064 + if (text === "First.\n\nWaiting on the build.\n\n")
4065 + yield* Deferred.succeed(nextParagraphDelivered, undefined);
4066 + }
4067 + })
4068 + : Effect.void,
4069 + }).pipe(
4070 + Effect.provideService(Clock.Clock, {
4071 + ...clock,
4072 + sleep: (duration) =>
4073 + clock.sleep(wakesEarly ? Duration.subtract(duration, Duration.millis(1)) : duration),
4074 + }),
4075 + Effect.forkChild,
4076 + );
4077 + // Waking a sleep runs ingestion until it suspends, so an idle write has landed by now.
4078 + const lastShown = Ref.get(delivered).pipe(Effect.map((shown) => shown.at(-1)));
4079 +
4080 + yield* Queue.offerAll(providerEvents, assistantText("First.\n\nWaiting on the build", true));
4081 + yield* Deferred.await(paragraphDelivered);
4082 + // The provider keeps its prompt open while a backgrounded command runs.
4083 + yield* TestClock.adjust("2 seconds");
4084 + assert.strictEqual(
4085 + yield* lastShown,
4086 + 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build"',
4087 + );
4088 + // The next paragraph's tail still flushes on a later pause.
4089 + yield* Queue.offerAll(
4090 + providerEvents,
4091 + assistantText("First.\n\nWaiting on the build.\n\nRunning", true),
4092 + );
4093 + yield* Deferred.await(nextParagraphDelivered);
4094 + yield* TestClock.adjust("1 second");
4095 + assert.strictEqual(
4096 + yield* lastShown,
4097 + 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\nRunning"',
4098 + );
4099 + yield* Queue.offerAll(providerEvents, [
4100 + ...assistantText("First.\n\nWaiting on the build.\n\nRunning tests", true),
4101 + blocker,
4102 + ...assistantText("First.\n\nWaiting on the build.\n\nRunning tests passed.", false),
4103 + ]);
4104 + yield* Deferred.await(blockerStarted);
4105 + // The appended text comes due while its final snapshots wait behind the blocked write.
4106 + yield* TestClock.adjust("1 second");
4107 + yield* Deferred.succeed(releaseBlocker, undefined);
4108 + yield* Queue.offer(providerEvents, rootTerminalEvent(ids, "completed"));
4109 + yield* Fiber.join(scenario);
4110 +
4111 + // Snapshots share the batch commit; the idle write commits alone, before it counts as shown.
4112 + // The final snapshots win over the stale append that came due behind them.
4113 + assert.deepEqual(yield* Ref.get(delivered), [
4114 + `message.updated:streaming:"First.\\n\\n" in commit`,
4115 + `turn_item.updated:streaming:"First.\\n\\n" in commit`,
4116 + 'message.updated:streaming:"First.\\n\\nWaiting on the build"',
4117 + 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build"',
4118 + `message.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\n" in commit`,
4119 + `turn_item.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\n" in commit`,
4120 + 'message.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\nRunning"',
4121 + 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\nRunning"',
4122 + `message.updated:final:"First.\\n\\nWaiting on the build.\\n\\nRunning tests passed." in commit`,
4123 + `turn_item.updated:final:"First.\\n\\nWaiting on the build.\\n\\nRunning tests passed." in commit`,
4124 + ]);
4125 + }),
4126 +);
4127 +
4128 +const heldChildText = "partial response";
4129 +
4130 +function childAssistantSnapshot(
4131 + ids: BackgroundScenarioIds,
4132 +): ProviderAdapter.ProviderAdapterV2Event {
4133 + return {
4134 + type: "turn_item.updated",
4135 + driver,
4136 + turnItem: {
4137 + ...makeLinkedChildTurnItemFixture({ ids, driver, type: "assistant_message" }),
4138 + text: heldChildText,
4139 + },
4140 + } as ProviderAdapter.ProviderAdapterV2Event;
4141 +}
4142 +
4143 +function linkedSubagentEvent(
4144 + ids: BackgroundScenarioIds,
4145 + status: "running" | "completed",
4146 +): ProviderAdapter.ProviderAdapterV2Event {
4147 + return {
4148 + type: "subagent.updated",
4149 + driver,
4150 + subagent: {
4151 + ...makeRunOwnedSubagentFixture({
4152 + ids,
4153 + providerInstanceId: ProviderInstanceId.make("codex"),
4154 + childThreadId: ids.childThreadId,
4155 + driver,
4156 + status: "running",
4157 + }),
4158 + status,
4159 + },
4160 + };
4161 +}
4162 +
4163 +const isHeldChildSnapshot = (event: ProviderAdapter.ProviderAdapterV2Event) =>
4164 + event.type === "turn_item.updated" &&
4165 + "text" in event.turnItem &&
4166 + event.turnItem.text === heldChildText;
4167 +
4168 +// Feeds a held child snapshot, then waits until ingestion has read past it.
4169 +const holdChildSnapshot = Effect.fn(function* (
4170 + ids: BackgroundScenarioIds,
4171 + events: Queue.Enqueue<ProviderAdapter.ProviderAdapterV2Event, Cause.Done>,
4172 + subagentIngested: Deferred.Deferred<void>,
4173 +) {
4174 + yield* Queue.offerAll(events, [
4175 + childThreadCreatedEvent(ids),
4176 + childAssistantSnapshot(ids),
4177 + linkedSubagentEvent(ids, "running"),
4178 + ]);
4179 + yield* Deferred.await(subagentIngested);
4180 +});
4181 +
4182 +it.effect("settles a child message only an idle flush wrote when the root is interrupted", () =>
4183 + Effect.gen(function* () {
4184 + const ids = backgroundScenarioIds("idle-child-interrupt");
4185 + const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event, Cause.Done>();
4186 + const subagentIngested = yield* Deferred.make<void>();
4187 + const flushed = yield* Deferred.make<void>();
4188 + const finalized = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);
4189 + const scenario = yield* runBackgroundItemScenario(
4190 + "idle-child-interrupt",
4191 + () => Stream.fromQueue(events),
4192 + {
4193 + onIngest: (event) =>
4194 + event.type === "subagent.updated"
4195 + ? Deferred.succeed(subagentIngested, undefined)
4196 + : isHeldChildSnapshot(event)
4197 + ? Deferred.succeed(flushed, undefined)
4198 + : Effect.void,
4199 + onFinalized: (written) => Ref.set(finalized, written),
4200 + },
4201 + ).pipe(Effect.forkChild);
4202 +
4203 + yield* holdChildSnapshot(ids, events, subagentIngested);
4204 + yield* TestClock.adjust("1 second");
4205 + yield* Deferred.await(flushed);
4206 + yield* Queue.offer(events, rootTerminalEvent(ids, "interrupted"));
4207 + yield* Fiber.join(scenario);
4208 +
4209 + const settled = (yield* Ref.get(finalized)).flatMap((event) =>
4210 + event.type === "turn-item.updated" && event.payload.id === ids.childItemId
4211 + ? [event.payload]
4212 + : [],
4213 + );
4214 + assert.equal(settled.length, 1);
4215 + assert.equal(settled[0]?.status, "interrupted");
4216 + assert.equal("text" in settled[0]! ? settled[0].text : null, heldChildText);
4217 + assert.isFalse("streaming" in settled[0]! && settled[0].streaming);
4218 + }),
4219 +);
4220 +
4221 +it.effect.each([
4222 + { name: "completed", terminalStatus: "completed" as const },
4223 + { name: "failed with its children continuing", terminalStatus: "failed" as const },
4224 +])("idle-flushes a continuing child's text after the root $name", ({ terminalStatus }) =>
4225 + Effect.gen(function* () {
4226 + const key = `idle-child-after-${terminalStatus}`;
4227 + const ids = backgroundScenarioIds(key);
4228 + const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event, Cause.Done>();
4229 + const subagentIngested = yield* Deferred.make<void>();
4230 + const rootFinalized = yield* Deferred.make<void>();
4231 + const flushed = yield* Deferred.make<void>();
4232 + const rootWrites = yield* Ref.make<ReadonlyArray<string>>([]);
4233 + const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(events), {
4234 + onIngest: (event) =>
4235 + event.type === "subagent.updated"
4236 + ? Deferred.succeed(subagentIngested, undefined)
4237 + : isHeldChildSnapshot(event)
4238 + ? Deferred.succeed(flushed, undefined)
4239 + : event.type === "message.updated" && event.message.threadId === ids.threadId
4240 + ? Ref.update(rootWrites, (current) => [...current, event.message.text])
4241 + : Effect.void,
4242 + onFinalized: () => Deferred.succeed(rootFinalized, undefined),
4243 + }).pipe(Effect.forkChild);
4244 +
4245 + yield* holdChildSnapshot(ids, events, subagentIngested);
4246 + // The root ends mid-paragraph: its tail is held, and the run settles without it.
4247 + yield* Queue.offer(
4248 + events,
4249 + streamedMessageEvent({
4250 + id: `message:${key}:root`,
4251 + threadId: ids.threadId,
4252 + runId: ids.runId,
4253 + role: "assistant",
4254 + text: "Root.\n\nhalf",
4255 + streaming: true,
4256 + }),
4257 + );
4258 + const terminal = rootTerminalEvent(ids, terminalStatus);
4259 + yield* Queue.offer(
4260 + events,
4261 + terminalStatus === "failed"
4262 + ? ({ ...terminal, childTurnsContinue: true } as ProviderAdapter.ProviderAdapterV2Event)
4263 + : terminal,
4264 + );
4265 + yield* Deferred.await(rootFinalized);
4266 + yield* TestClock.adjust("1 second");
4267 + yield* Deferred.await(flushed);
4268 + yield* Queue.offer(events, linkedSubagentEvent(ids, "completed"));
4269 + const observed = yield* Fiber.join(scenario);
4270 +
4271 + assert.deepEqual(observed.slice(-3), [
4272 + "root-finalized",
4273 + "turn_item:running",
4274 + "subagent:completed",
4275 + ]);
4276 + // A streaming write after the run settled would reopen the root's message.
4277 + assert.deepEqual(yield* Ref.get(rootWrites), ["Root.\n\n"]);
4278 + }),
4279 +);
4280 +
4281 +function streamedMessageEvent(input: {
4282 + readonly id: string;
4283 + readonly threadId: ThreadId;
4284 + readonly runId: RunId | null;
4285 + readonly role: "assistant" | "user";
4286 + readonly text: string;
4287 + readonly streaming: boolean;
4288 +}): ProviderAdapter.ProviderAdapterV2Event {
4289 + const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z");
4290 + return {
4291 + type: "message.updated",
4292 + driver,
4293 + message: {
4294 + createdBy: input.role === "user" ? "user" : "agent",
4295 + creationSource: "provider",
4296 + id: MessageId.make(input.id),
4297 + threadId: input.threadId,
4298 + runId: input.runId,
4299 + nodeId: null,
4300 + role: input.role,
4301 + text: input.text,
4302 + attachments: [],
4303 + streaming: input.streaming,
4304 + createdAt: now,
4305 + updatedAt: now,
4306 + },
4307 + } as ProviderAdapter.ProviderAdapterV2Event;
4308 +}
4309 +
4310 +it.effect("drops a child's held text once the child is handed to another run", () =>
4311 + Effect.gen(function* () {
4312 + const key = "idle-child-handoff";
4313 + const ids = backgroundScenarioIds(key);
4314 + const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event, Cause.Done>();
4315 + const subagentIngested = yield* Deferred.make<void>();
4316 + const handoffRead = yield* Deferred.make<void>();
4317 + const rootFlushed = yield* Deferred.make<void>();
4318 + const staleWrites = yield* Ref.make<ReadonlyArray<string>>([]);
4319 + const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(events), {
4320 + onIngest: (event) => {
4321 + if (event.type === "subagent.updated") return Deferred.succeed(subagentIngested, undefined);
4322 + if (event.type !== "message.updated" && event.type !== "turn_item.updated")
4323 + return Effect.void;
4324 + const message = event.type === "message.updated" ? event.message : event.turnItem;
4325 + if (message.threadId === ids.childThreadId)
4326 + return Ref.update(staleWrites, (current) => [...current, event.type]);
4327 + if (event.type === "message.updated" && event.message.role === "user")
4328 + return Deferred.succeed(handoffRead, undefined);
4329 + return Deferred.succeed(rootFlushed, undefined);
4330 + },
4331 + }).pipe(Effect.forkChild);
4332 +
4333 + yield* holdChildSnapshot(ids, events, subagentIngested);
4334 + const handoff = linkedSubagentEvent(ids, "running");
4335 + yield* Queue.offerAll(events, [
4336 + streamedMessageEvent({
4337 + id: `message:${key}:child`,
4338 + threadId: ids.childThreadId,
4339 + runId: null,
4340 + role: "assistant",
4341 + text: heldChildText,
4342 + streaming: true,
4343 + }),
4344 + handoff.type === "subagent.updated"
4345 + ? { ...handoff, subagent: { ...handoff.subagent, runId: RunId.make(`run:${key}:next`) } }
4346 + : handoff,
4347 + // Due in the same flush as the child's text, after it.
4348 + streamedMessageEvent({
4349 + id: `message:${key}:root`,
4350 + threadId: ids.threadId,
4351 + runId: ids.runId,
4352 + role: "assistant",
4353 + text: "Root partial",
4354 + streaming: true,
4355 + }),
4356 + streamedMessageEvent({
4357 + id: `message:${key}:marker`,
4358 + threadId: ids.threadId,
4359 + runId: ids.runId,
4360 + role: "user",
4361 + text: "marker",
4362 + streaming: false,
4363 + }),
4364 + ]);
4365 + yield* Deferred.await(handoffRead);
4366 + yield* TestClock.adjust("1 second");
4367 + yield* Deferred.await(rootFlushed);
4368 + yield* Queue.end(events);
4369 + yield* Fiber.join(scenario);
4370 +
4371 + assert.deepEqual(yield* Ref.get(staleWrites), []);
4372 + }),
4373 +);
apps/server/src/orchestration-v2/RunExecutionService.ts
+147 -60
@@ -32,9 +32,11 @@ import * as Effect from "effect/Effect";
32 import * as Exit from "effect/Exit";
33 import * as Fiber from "effect/Fiber";
34 import * as Layer from "effect/Layer";
35 +import * as Queue from "effect/Queue";
36 import * as Ref from "effect/Ref";
37 import * as Schedule from "effect/Schedule";
38 import * as Schema from "effect/Schema";
39 +import * as Stream from "effect/Stream";
40
41 import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts";
42 import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts";
@@ -174,6 +176,12 @@ function isRunOwnedSubagentTerminalStatus(
176 return status === "interrupted" || status === "failed" || status === "cancelled";
177 }
178
179 +function streamedThreadId(event: ProviderAdapter.ProviderAdapterV2Event): ThreadId | null {
180 + if (event.type === "message.updated") return event.message.threadId;
181 + if (event.type === "turn_item.updated") return event.turnItem.threadId;
182 + return null;
183 +}
184 +
185 function childTurnsContinue(terminal: ProviderTerminalEvent): boolean {
186 return terminal.status === "failed" && terminal.childTurnsContinue === true;
187 }
@@ -1073,6 +1081,12 @@ export const layer: Layer.Layer<
1081 ReadonlySet<OrchestrationV2TurnItem["id"]>
1082 >(new Set(inheritedBackgroundTurnItemsById.keys()));
1083 const openRunOwnedSubagents = yield* Ref.make(emptyOpenRunOwnedSubagentProjection());
1084 + // The last stored root snapshot's pendingBackgroundWork; adapters that omit it leave it false.
1085 + let rootSnapshotPendingWork = false;
1086 + const assistantStreaming = makeAssistantStreamingFilter(responseStreamingMode);
1087 + // Set when ingestion ends or finalization settles the run's open items, which a later
1088 + // streaming snapshot must not reopen.
1089 + let idleFlushClosed = false;
1090 const finalizeRootRun = Effect.fn("RunExecutionService.finalizeRootRun")(function* (
1091 terminal: ProviderTerminalEvent,
1092 ) {
@@ -1115,6 +1129,7 @@ export const layer: Layer.Layer<
1129 );
1130 if (isRunOwnedSubagentTerminalStatus(terminal.status) && !childTurnsContinue(terminal)) {
1131 yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection());
1132 + idleFlushClosed = true;
1133 }
1134 yield* Ref.set(rootRunFinalized, true);
1135 });
@@ -1301,6 +1316,10 @@ export const layer: Layer.Layer<
1316 if (yield* Ref.get(providerThreadOwnerLost)) {
1317 return true;
1318 }
1319 + // The live probe reads adapter state, which can be ahead of the events consumed here.
1320 + if (rootSnapshotPendingWork) {
1321 + return false;
1322 + }
1323 // Claude background Bash has no turn-item projection. Use only the
1324 // thread-scoped probe: session-wide pending work (siblings, wake
1325 // buffers, session subagents) must not pin this root subscription.
@@ -1315,7 +1334,6 @@ export const layer: Layer.Layer<
1334 }
1335 return true;
1336 });
1318 - const filterAssistantEvent = makeAssistantStreamingFilter(responseStreamingMode);
1337 // Held text writes nothing, so its item would take a position only
1338 // when finally written, behind every item that began after it.
1339 const reservedHeldTurnItemIds = new Set<TurnItemId>();
@@ -1398,6 +1416,121 @@ export const layer: Layer.Layer<
1416 }
1417 return { event, accepted };
1418 });
1419 + // Per-event ingestion avoids adding a span for every provider snapshot.
1420 + const ingestDelivered = Effect.fnUntraced(function* (
1421 + event: ProviderAdapter.ProviderAdapterV2Event,
1422 + deliveredEvent: ProviderAdapter.ProviderAdapterV2Event,
1423 + ) {
1424 + const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen);
1425 + const isRootProviderThreadUpdate =
1426 + event.type === "provider_thread.updated" &&
1427 + event.providerThread.id === input.providerThread.id;
1428 + const nativeChildTurn =
1429 + event.type === "provider_turn.updated" &&
1430 + event.providerTurn.providerThreadId !== input.providerThread.id;
1431 + if (event.type === "provider_turn.updated" && !nativeChildTurn) {
1432 + firstRootProviderTurnId ??= event.providerTurn.id;
1433 + }
1434 + const storedEvents = yield* providerEventIngestor.ingestNormalized({
1435 + analyticsContext: nativeChildTurn
1436 + ? {
1437 + ...analyticsContext,
1438 + origin: NATIVE_SUBAGENT_TURN_ORIGIN,
1439 + segment: "native_child",
1440 + }
1441 + : {
1442 + ...analyticsContext,
1443 + segment:
1444 + event.type === "provider_turn.updated" &&
1445 + event.providerTurn.id !== firstRootProviderTurnId
1446 + ? "continuation"
1447 + : input.attempt.reason,
1448 + },
1449 + providerSessionId: input.providerSessionId,
1450 + providerInstanceId: input.run.providerInstanceId,
1451 + threadId: input.run.threadId,
1452 + runId: input.run.id,
1453 + nodeId: input.rootNode.id,
1454 + event: deliveredEvent,
1455 + ...(isRootProviderThreadUpdate
1456 + ? rootTerminalAlreadySeen
1457 + ? {
1458 + writeIfProviderThreadOwner: {
1459 + providerThreadId: input.providerThread.id,
1460 + runId: input.run.id,
1461 + activeAttemptId: input.attempt.id,
1462 + expectedLastRunOrdinal: input.run.ordinal,
1463 + },
1464 + }
1465 + : {
1466 + writeIfRunCurrent: {
1467 + runId: input.run.id,
1468 + activeAttemptId: input.attempt.id,
1469 + expectedStatus: "running" as const,
1470 + },
1471 + }
1472 + : {}),
1473 + });
1474 + if (isRootProviderThreadUpdate && rootTerminalAlreadySeen && storedEvents.length === 0) {
1475 + yield* Ref.set(providerThreadOwnerLost, true);
1476 + }
1477 + return storedEvents.length;
1478 + });
1479 + // The timer only wakes ingestion. The flush runs as an ingestion step outside any provider
1480 + // batch, so it stays ordered with routing and snapshots, and a write has committed when
1481 + // it returns.
1482 + const idleFlushTicks = yield* Queue.dropping<{
1483 + readonly type: "idle_flush";
1484 + readonly at: number;
1485 + }>(1);
1486 + let idleFlushWake: { readonly at: number; readonly fiber: Fiber.Fiber<void> } | undefined;
1487 + const scheduleIdleFlush = Effect.suspend(() => {
1488 + const at = assistantStreaming.nextIdleFlushAt();
1489 + if (
1490 + idleFlushClosed ||
1491 + at === null ||
1492 + (idleFlushWake !== undefined && idleFlushWake.at <= at)
1493 + )
1494 + return Effect.void;
1495 + const previous = idleFlushWake?.fiber;
1496 + return Effect.gen(function* () {
1497 + if (previous !== undefined) yield* Fiber.interrupt(previous);
1498 + const now = DateTime.toEpochMillis(yield* DateTime.now);
1499 + const fiber = yield* Effect.sleep(Duration.millis(Math.max(0, at - now))).pipe(
1500 + Effect.andThen(Queue.offer(idleFlushTicks, { type: "idle_flush", at })),
1501 + Effect.asVoid,
1502 + Effect.forkDetach,
1503 + );
1504 + idleFlushWake = { at, fiber };
1505 + });
1506 + });
1507 + const closeIdleFlush = Effect.suspend(() => {
1508 + idleFlushClosed = true;
1509 + const wake = idleFlushWake;
1510 + idleFlushWake = undefined;
1511 + return wake === undefined ? Effect.void : Fiber.interrupt(wake.fiber);
1512 + });
1513 + // Per-tick ingestion avoids adding a span for every idle wake.
1514 + const flushIdleAssistantText = Effect.fnUntraced(function* (wokeAt: number) {
1515 + // A tick means its timer slept until `wokeAt`, but the wall clock can still read up
1516 + // to a millisecond earlier; trusting it would keep the finished wake and never re-arm.
1517 + const now = Math.max(DateTime.toEpochMillis(yield* DateTime.now), wokeAt);
1518 + // A wake due by now has fired; a later deadline needs a new one.
1519 + if (idleFlushWake !== undefined && idleFlushWake.at <= now) idleFlushWake = undefined;
1520 + if (idleFlushClosed) return;
1521 + const rootSettled = yield* Ref.get(rootRunFinalized);
1522 + for (const event of assistantStreaming.takeIdleFlushes(now)) {
1523 + // Settling the root run closed its messages; only children may still stream.
1524 + if (rootSettled && streamedThreadId(event) === input.run.threadId) continue;
1525 + // The held text's child may have been handed to another run since it arrived.
1526 + if (!routeProviderEvent(event, routeIdentity, yield* Ref.get(eventRouting))[0])
1527 + continue;
1528 + // A failed write fails ingestion like any provider write.
1529 + yield* ingestDelivered(event, event);
1530 + yield* trackChildLifecycle(event, true);
1531 + }
1532 + yield* scheduleIdleFlush;
1533 + });
1534 const ingestEvent = Effect.fnUntraced(function* ({
1535 event,
1536 accepted,
@@ -1414,65 +1547,12 @@ export const layer: Layer.Layer<
1547 yield* Ref.set(latestRootProviderTurn, event.providerTurn);
1548 }
1549 let storedEventCount = 0;
1417 - const deliveredEvent = filterAssistantEvent(
1550 + const deliveredEvent = assistantStreaming.filter(
1551 event,
1552 DateTime.toEpochMillis(yield* DateTime.now),
1553 );
1554 if (deliveredEvent) {
1422 - const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen);
1423 - const isRootProviderThreadUpdate =
1424 - event.type === "provider_thread.updated" &&
1425 - event.providerThread.id === input.providerThread.id;
1426 - const nativeChildTurn =
1427 - event.type === "provider_turn.updated" &&
1428 - event.providerTurn.providerThreadId !== input.providerThread.id;
1429 - if (event.type === "provider_turn.updated" && !nativeChildTurn) {
1430 - firstRootProviderTurnId ??= event.providerTurn.id;
1431 - }
1432 - const storedEvents = yield* providerEventIngestor.ingestNormalized({
1433 - analyticsContext: nativeChildTurn
1434 - ? {
1435 - ...analyticsContext,
1436 - origin: NATIVE_SUBAGENT_TURN_ORIGIN,
1437 - segment: "native_child",
1438 - }
1439 - : {
1440 - ...analyticsContext,
1441 - segment:
1442 - event.type === "provider_turn.updated" &&
1443 - event.providerTurn.id !== firstRootProviderTurnId
1444 - ? "continuation"
1445 - : input.attempt.reason,
1446 - },
1447 - providerSessionId: input.providerSessionId,
1448 - providerInstanceId: input.run.providerInstanceId,
1449 - threadId: input.run.threadId,
1450 - runId: input.run.id,
1451 - nodeId: input.rootNode.id,
1452 - event: deliveredEvent,
1453 - ...(isRootProviderThreadUpdate
1454 - ? rootTerminalAlreadySeen
1455 - ? {
1456 - writeIfProviderThreadOwner: {
1457 - providerThreadId: input.providerThread.id,
1458 - runId: input.run.id,
1459 - activeAttemptId: input.attempt.id,
1460 - expectedLastRunOrdinal: input.run.ordinal,
1461 - },
1462 - }
1463 - : {
1464 - writeIfRunCurrent: {
1465 - runId: input.run.id,
1466 - activeAttemptId: input.attempt.id,
1467 - expectedStatus: "running" as const,
1468 - },
1469 - }
1470 - : {}),
1471 - });
1472 - storedEventCount = storedEvents.length;
1473 - if (isRootProviderThreadUpdate && rootTerminalAlreadySeen && storedEventCount === 0) {
1474 - yield* Ref.set(providerThreadOwnerLost, true);
1475 - }
1555 + storedEventCount = yield* ingestDelivered(event, deliveredEvent);
1556 } else if (
1557 event.type === "turn_item.updated" &&
1558 (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") &&
@@ -1489,6 +1569,7 @@ export const layer: Layer.Layer<
1569 if (event.type === "provider_thread.updated") {
1570 if (event.providerThread.id === input.providerThread.id && storedEventCount > 0) {
1571 yield* Ref.set(latestProviderThread, event.providerThread);
1572 + rootSnapshotPendingWork = event.pendingBackgroundWork === true;
1573 }
1574 }
1575 if (
@@ -1512,14 +1593,19 @@ export const layer: Layer.Layer<
1593 }
1594 }
1595 yield* trackChildLifecycle(event, deliveredEvent !== null);
1596 + yield* scheduleIdleFlush;
1597 });
1598 const providerEventFiber = yield* eventSink
1599 .consumeProviderEvents(
1518 - eventSubscription.events,
1600 + eventSubscription.events.pipe(
1601 + Stream.merge(Stream.fromQueue(idleFlushTicks), { haltStrategy: "left" }),
1602 + ),
1603 (event) =>
1520 - Effect.gen(function* () {
1521 - yield* ingestEvent(yield* routeEvent(event));
1522 - }),
1604 + event.type === "idle_flush"
1605 + ? flushIdleAssistantText(event.at)
1606 + : Effect.gen(function* () {
1607 + yield* ingestEvent(yield* routeEvent(event));
1608 + }),
1609 Effect.gen(function* () {
1610 if (yield* agentTurnsSettled) yield* endPreviewActivity;
1611 return yield* shouldStopProviderEventIngestion;
@@ -1527,6 +1613,7 @@ export const layer: Layer.Layer<
1613 Ref.get(rootTerminalSeen).pipe(Effect.map((seen) => !seen)),
1614 )
1615 .pipe(
1616 + Effect.ensuring(closeIdleFlush),
1617 Effect.mapError((cause) => new RunExecutionIngestError({ runId: input.run.id, cause })),
1618 Effect.flatMap(() =>
1619 Effect.gen(function* () {
apps/server/src/orchestration-v2/assistantStreaming.test.ts
+171 -2
@@ -1,5 +1,8 @@
1 -import { describe, expect, it } from "vite-plus/test";
2 -import { splitBufferedAssistantText } from "./assistantStreaming.ts";
1 +import { MessageId, ProviderDriverKind, ThreadId } from "@t3tools/contracts";
2 +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
3 +import * as DateTime from "effect/DateTime";
4 +import { assert, describe, expect, it } from "vite-plus/test";
5 +import { makeAssistantStreamingFilter, splitBufferedAssistantText } from "./assistantStreaming.ts";
6
7 describe("V2 assistant streaming", () => {
8 it("keeps code fences intact", () => {
@@ -24,3 +27,169 @@ it("holds a streamed section heading until its content has a boundary", () => {
27 rest: "Next",
28 });
29 });
30 +
31 +type Step =
32 + // A snapshot at `at` ms, and the text it delivers at once (null when held back).
33 + | { readonly at: number; readonly text: string; readonly shows: string | null }
34 + | { readonly at: number; readonly final: string }
35 + | { readonly id: string; readonly at: number; readonly text: string; readonly shows: string }
36 + // Held snapshots due by `idle` ms, delivered whole.
37 + | { readonly idle: number; readonly shows: ReadonlyArray<string> }
38 + | { readonly dueAt: number | null };
39 +
40 +const createdAt = DateTime.makeUnsafe(0);
41 +const snapshot = (id: string, text: string, streaming: boolean) =>
42 + ({
43 + type: "message.updated",
44 + driver: ProviderDriverKind.make("codex"),
45 + message: {
46 + createdBy: "agent",
47 + creationSource: "provider",
48 + id: MessageId.make(id),
49 + threadId: ThreadId.make("thread"),
50 + runId: null,
51 + nodeId: null,
52 + role: "assistant",
53 + text,
54 + attachments: [],
55 + streaming,
56 + createdAt,
57 + updatedAt: createdAt,
58 + },
59 + }) satisfies ProviderAdapter.ProviderAdapterV2Event;
60 +const shown = (event: ProviderAdapter.ProviderAdapterV2Event | null) =>
61 + event?.type === "message.updated" ? event.message.text : null;
62 +
63 +it.each<{ readonly name: string; readonly mode?: "turn"; readonly steps: ReadonlyArray<Step> }>([
64 + {
65 + name: "an unfinished code fence",
66 + steps: [
67 + { at: 0, text: "A\n\n```ts\nx()", shows: "A\n\n" },
68 + { dueAt: 800 },
69 + { idle: 799, shows: [] },
70 + { idle: 800, shows: ["A\n\n```ts\nx()"] },
71 + { dueAt: null },
72 + { at: 1_300, text: "A\n\n```ts\nx()\n```\n\nB", shows: "A\n\n```ts\nx()\n```\n\n" },
73 + ],
74 + },
75 + {
76 + name: "a list's partial last item",
77 + steps: [
78 + { at: 0, text: "Steps:\n\n- one\n- tw", shows: "Steps:\n\n- one\n" },
79 + { idle: 800, shows: ["Steps:\n\n- one\n- tw"] },
80 + { at: 1_300, text: "Steps:\n\n- one\n- two\n- th", shows: "Steps:\n\n- one\n- two\n" },
81 + ],
82 + },
83 + {
84 + name: "an incomplete table row",
85 + steps: [
86 + { at: 0, text: "Intro.\n\n| a | b |\n|---|---|\n| 1 |", shows: "Intro.\n\n" },
87 + { idle: 800, shows: ["Intro.\n\n| a | b |\n|---|---|\n| 1 |"] },
88 + {
89 + at: 1_300,
90 + text: "Intro.\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\nAfter",
91 + shows: "Intro.\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\n",
92 + },
93 + ],
94 + },
95 + {
96 + name: "more of the same paragraph after an idle delivery",
97 + steps: [
98 + { at: 0, text: "A\n\nB", shows: "A\n\n" },
99 + { idle: 800, shows: ["A\n\nB"] },
100 + { at: 900, text: "A\n\nB more", shows: null },
101 + { idle: 1_699, shows: [] },
102 + { idle: 1_700, shows: ["A\n\nB more"] },
103 + { idle: 5_000, shows: [] },
104 + // Its only boundary is behind what the idle delivery showed.
105 + { at: 5_100, text: "A\n\nB more and", shows: null },
106 + { at: 5_200, text: "A\n\nB more and.\n\nC", shows: "A\n\nB more and.\n\n" },
107 + { at: 5_300, final: "A\n\nB more and.\n\nC." },
108 + { idle: 10_000, shows: [] },
109 + ],
110 + },
111 + {
112 + name: "completion at the idle deadline",
113 + steps: [
114 + { at: 0, text: "A\n\nB", shows: "A\n\n" },
115 + { at: 800, final: "A\n\nB." },
116 + { idle: 800, shows: [] },
117 + ],
118 + },
119 + {
120 + name: "completion right after an idle delivery at the same instant",
121 + steps: [
122 + { at: 0, text: "A\n\nB", shows: "A\n\n" },
123 + { idle: 800, shows: ["A\n\nB"] },
124 + { at: 800, final: "A\n\nB." },
125 + { idle: 5_000, shows: [] },
126 + ],
127 + },
128 + {
129 + name: "several fast paragraphs inside the throttle",
130 + steps: [
131 + { at: 0, text: "A\n\nB", shows: "A\n\n" },
132 + { at: 100, text: "A\n\nB\n\nC", shows: null },
133 + { at: 200, text: "A\n\nB\n\nC\n\nD", shows: null },
134 + { at: 300, text: "A\n\nB\n\nC\n\nD\n\nE", shows: null },
135 + { idle: 1_099, shows: [] },
136 + { idle: 1_100, shows: ["A\n\nB\n\nC\n\nD\n\nE"] },
137 + ],
138 + },
139 + {
140 + name: "two messages due at different times",
141 + steps: [
142 + { id: "a", at: 0, text: "A\n\nx", shows: "A\n\n" },
143 + { id: "b", at: 300, text: "B\n\ny", shows: "B\n\n" },
144 + { dueAt: 800 },
145 + { idle: 800, shows: ["A\n\nx"] },
146 + { dueAt: 1_100 },
147 + { idle: 1_100, shows: ["B\n\ny"] },
148 + ],
149 + },
150 + {
151 + name: "empty text",
152 + steps: [{ at: 0, text: "", shows: null }, { dueAt: null }],
153 + },
154 + {
155 + name: "whitespace-only text",
156 + steps: [
157 + { at: 0, text: " \n", shows: null },
158 + { idle: 800, shows: [" \n"] },
159 + { idle: 5_000, shows: [] },
160 + ],
161 + },
162 + {
163 + // Rewrites replace delivered text: shorter at once, same length after the throttle.
164 + name: "provider rewrites and truncation",
165 + steps: [
166 + { at: 0, text: "Original long paragraph.\n\n", shows: "Original long paragraph.\n\n" },
167 + { at: 1_000, text: "New.\n\n", shows: "New.\n\n" },
168 + { at: 1_100, text: "Old.\n\n", shows: null },
169 + { idle: 1_900, shows: ["Old.\n\n"] },
170 + { at: 2_500, text: "Old", shows: null },
171 + { idle: 3_300, shows: ["Old"] },
172 + { at: 4_000, text: "", shows: null },
173 + { idle: 4_800, shows: [""] },
174 + ],
175 + },
176 + {
177 + name: "turn mode",
178 + mode: "turn",
179 + steps: [{ at: 0, text: "A\n\nB", shows: null }, { dueAt: null }],
180 + },
181 +])("paragraph streaming holds and idle-delivers $name", ({ mode, steps }) => {
182 + const filter = makeAssistantStreamingFilter(mode ?? "paragraph", 800);
183 + for (const step of steps) {
184 + if ("dueAt" in step) assert.equal(filter.nextIdleFlushAt(), step.dueAt);
185 + else if ("idle" in step)
186 + assert.deepEqual(filter.takeIdleFlushes(step.idle).map(shown), [...step.shows]);
187 + else if ("final" in step)
188 + assert.equal(shown(filter.filter(snapshot("m", step.final, false), step.at)), step.final);
189 + else
190 + assert.equal(
191 + shown(filter.filter(snapshot("id" in step ? step.id : "m", step.text, true), step.at)),
192 + step.shows,
193 + );
194 + }
195 +});
apps/server/src/orchestration-v2/assistantStreaming.ts
+124 -39
@@ -68,44 +68,129 @@ export function splitBufferedAssistantText(text: string): { ready: string; rest:
68 return { ready: text.slice(0, boundary), rest: text.slice(boundary) };
69 }
70
71 -export function makeAssistantStreamingFilter(mode: ResponseStreamingMode) {
72 - const delivered = new Map<string, { text: string; at: number }>();
73 - return (
74 - event: ProviderAdapter.ProviderAdapterV2Event,
75 - now: number,
76 - ): ProviderAdapter.ProviderAdapterV2Event | null => {
77 - if (
78 - event.type === "node.updated" &&
79 - event.node.kind === "assistant_message" &&
80 - event.node.status === "running"
81 - )
82 - return null;
83 - const message =
84 - event.type === "message.updated" && event.message.role === "assistant"
85 - ? event.message
86 - : event.type === "turn_item.updated" &&
87 - (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning")
88 - ? event.turnItem
89 - : null;
90 - if (!message) return event;
91 - const key = `${event.type}:${message.id}`;
92 - if (!message.streaming) {
93 - delivered.delete(key);
94 - return event;
95 - }
96 - if (mode === "turn") return null;
97 - const previous = delivered.get(key);
98 - if (previous && now - previous.at < 400) return null;
99 - const { ready } = splitBufferedAssistantText(message.text);
100 - if (!ready || ready === previous?.text) return null;
101 - delivered.set(key, { text: ready, at: now });
102 - if (event.type === "message.updated")
103 - return { ...event, message: { ...event.message, text: ready } };
104 - if (
105 - event.type === "turn_item.updated" &&
106 - (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning")
107 - )
108 - return { ...event, turnItem: { ...event.turnItem, text: ready } };
109 - return event;
71 +// Long enough to ride out normal gaps between chunks; short enough that text written before a
72 +// long tool call or background wait does not look lost.
73 +const ASSISTANT_IDLE_FLUSH_MS = 800;
74 +
75 +type StreamedMessage = {
76 + readonly id: string;
77 + readonly text: string;
78 + readonly streaming: boolean;
79 +};
80 +
81 +interface StreamTrack {
82 + readonly text: string;
83 + readonly at: number;
84 + // Latest snapshot whose text has not all been delivered yet.
85 + readonly held: ProviderAdapter.ProviderAdapterV2Event | null;
86 + readonly dueAt: number;
87 +}
88 +
89 +function streamedMessage(event: ProviderAdapter.ProviderAdapterV2Event): StreamedMessage | null {
90 + if (event.type === "message.updated" && event.message.role === "assistant") return event.message;
91 + if (
92 + event.type === "turn_item.updated" &&
93 + (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning")
94 + )
95 + return event.turnItem;
96 + return null;
97 +}
98 +
99 +function withText(
100 + event: ProviderAdapter.ProviderAdapterV2Event,
101 + text: string,
102 +): ProviderAdapter.ProviderAdapterV2Event {
103 + if (event.type === "message.updated") return { ...event, message: { ...event.message, text } };
104 + if (
105 + event.type === "turn_item.updated" &&
106 + (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning")
107 + )
108 + return { ...event, turnItem: { ...event.turnItem, text } };
109 + return event;
110 +}
111 +
112 +/**
113 + * Paragraph mode delivers streamed text at Markdown boundaries, at most every 400ms. A snapshot
114 + * held back that way is delivered whole once its message has been idle for `idleFlushMs`, so a
115 + * provider that pauses mid-turn still shows everything it wrote.
116 + */
117 +export function makeAssistantStreamingFilter(
118 + mode: ResponseStreamingMode,
119 + idleFlushMs = ASSISTANT_IDLE_FLUSH_MS,
120 +) {
121 + const tracks = new Map<string, StreamTrack>();
122 + // Earliest deadline of any held snapshot, or earlier: a newer snapshot only pushes its own
123 + // deadline later, so the cache stays O(1) per event and is recomputed when it comes due.
124 + let nextDueAt: number | null = null;
125 + const track = (key: string, value: StreamTrack) => {
126 + tracks.set(key, value);
127 + if (value.held !== null && (nextDueAt === null || value.dueAt < nextDueAt))
128 + nextDueAt = value.dueAt;
129 + };
130 + return {
131 + filter(
132 + event: ProviderAdapter.ProviderAdapterV2Event,
133 + now: number,
134 + ): ProviderAdapter.ProviderAdapterV2Event | null {
135 + if (
136 + event.type === "node.updated" &&
137 + event.node.kind === "assistant_message" &&
138 + event.node.status === "running"
139 + )
140 + return null;
141 + const message = streamedMessage(event);
142 + if (!message) return event;
143 + const key = `${event.type}:${message.id}`;
144 + if (!message.streaming) {
145 + tracks.delete(key);
146 + return event;
147 + }
148 + if (mode === "turn") return null;
149 + const previous = tracks.get(key);
150 + const delivered = previous?.text ?? "";
151 + const hold = () => {
152 + track(key, {
153 + text: delivered,
154 + at: previous?.at ?? Number.NEGATIVE_INFINITY,
155 + held: message.text !== delivered ? event : null,
156 + dueAt: now + idleFlushMs,
157 + });
158 + return null;
159 + };
160 + if (previous && now - previous.at < 400) return hold();
161 + const { ready } = splitBufferedAssistantText(message.text);
162 + // An idle delivery may already show more than the next boundary; never shrink it. A
163 + // provider rewrite or truncation is not a prefix and replaces it.
164 + if (delivered.startsWith(ready)) return hold();
165 + track(key, {
166 + text: ready,
167 + at: now,
168 + held: ready !== message.text ? event : null,
169 + dueAt: now + idleFlushMs,
170 + });
171 + return withText(event, ready);
172 + },
173 + nextIdleFlushAt: (): number | null => nextDueAt,
174 + /** Returns the held snapshots due by `now` and counts them as delivered. */
175 + takeIdleFlushes(now: number): ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event> {
176 + const due: Array<ProviderAdapter.ProviderAdapterV2Event> = [];
177 + nextDueAt = null;
178 + for (const [key, current] of tracks) {
179 + const event = current.held;
180 + if (event === null) continue;
181 + if (current.dueAt > now) {
182 + if (nextDueAt === null || current.dueAt < nextDueAt) nextDueAt = current.dueAt;
183 + continue;
184 + }
185 + due.push(event);
186 + tracks.set(key, {
187 + text: streamedMessage(event)?.text ?? current.text,
188 + at: now,
189 + held: null,
190 + dueAt: now,
191 + });
192 + }
193 + return due;
194 + },
195 };
196 }