@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 5,237 lines 196 KB
Raw
1 import { assert, it } from "@effect/vitest";
2 import {
3 CheckpointScopeId,
4 ChatAttachmentId,
5 ChatFileAttachment,
6 CommandId,
7 MessageId,
8 NodeId,
9 type OrchestrationV2AppThread,
10 type OrchestrationV2CheckpointScope,
11 type OrchestrationV2DomainEvent,
12 type OrchestrationV2ExecutionNode,
13 type OrchestrationV2PendingBackgroundTask,
14 type OrchestrationV2ProviderThread,
15 type OrchestrationV2ProviderTurn,
16 type OrchestrationV2Run,
17 type OrchestrationV2RunAttempt,
18 type OrchestrationV2StoredEvent,
19 type OrchestrationV2Subagent,
20 type OrchestrationV2TurnItem,
21 ProviderDriverKind,
22 ProviderInstanceId,
23 ProviderSessionId,
24 ProviderThreadId,
25 ProviderTurnId,
26 RunAttemptId,
27 RunId,
28 ServerSettingsError,
29 ThreadId,
30 TurnItemId,
31 } from "@t3tools/contracts";
32 import * as Cause from "effect/Cause";
33 import * as Clock from "effect/Clock";
34 import * as Exit from "effect/Exit";
35 import * as Fiber from "effect/Fiber";
36 import * as DateTime from "effect/DateTime";
37 import * as Deferred from "effect/Deferred";
38 import * as Duration from "effect/Duration";
39 import * as Effect from "effect/Effect";
40 import * as Layer from "effect/Layer";
41 import * as Option from "effect/Option";
42 import * as Queue from "effect/Queue";
43 import * as Ref from "effect/Ref";
44 import * as Stream from "effect/Stream";
45 import * as TestClock from "effect/testing/TestClock";
46 import { CodexAppServerRequestError } from "effect-codex-app-server/errors";
47 import { SqlError, classifySqliteError } from "effect/sql/SqlError";
48
49 import { toPersistenceSqlError } from "../persistence/Errors.ts";
50 import * as ServerSettings from "../serverSettings.ts";
51 import * as CheckpointService from "./CheckpointService.ts";
52 import type { PendingOrchestrationEffectV2 } from "./EffectOutbox.ts";
53 import * as EventSink from "./EventSink.ts";
54 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
55 import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
56 import * as ProjectionStore from "./ProjectionStore.ts";
57 import * as ProviderCommitBatch from "./ProviderCommitBatch.ts";
58 import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
59 import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts";
60 import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts";
61 import * as PreviewAgentActivity from "../preview/PreviewAgentActivity.ts";
62 import * as RunExecutionService from "./RunExecutionService.ts";
63 import * as RunFinalizationService from "./RunFinalizationService.ts";
64 import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
65 import * as TurnItemPositionStore from "./TurnItemPositionStore.ts";
66
67 const mockEventSink = (implementation: Partial<EventSink.EventSinkV2["Service"]>) =>
68 Layer.mock(EventSink.EventSinkV2)({
69 consumeProviderEvents: (source, process, after) =>
70 source.pipe(
71 Stream.mapEffect((event) => process(event).pipe(Effect.andThen(after))),
72 Stream.takeUntil(Boolean),
73 Stream.runDrain,
74 ),
75 ...implementation,
76 });
77
78 const driver = ProviderDriverKind.make("codex");
79 const rootAppThread = (id: ThreadId) =>
80 ({
81 id,
82 lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: id },
83 }) as OrchestrationV2AppThread;
84
85 const layerTurnItemPositionsTest = Layer.mock(TurnItemPositionStore.TurnItemPositionStoreV2)({
86 allocate: () => Effect.succeed(1),
87 });
88
89 const layerRunExecutionTest = RunExecutionService.layer.pipe(
90 Layer.provide(
91 Layer.mergeAll(
92 McpAppModelContext.layerEmpty,
93 PreviewAgentActivity.layer,
94 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
95 layerTurnItemPositionsTest,
96 mockEventSink({}),
97 IdAllocator.layer,
98 ThreadCommandExecutor.layer,
99 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
100 ingestNormalized: () => Effect.succeed([]),
101 }),
102 ServerSettings.layerTest(),
103 ),
104 ),
105 );
106
107 it.effect("routes shared-runtime events only to their owning root run", () =>
108 Effect.gen(function* () {
109 const now = yield* DateTime.now;
110 const first: RunExecutionService.ProviderEventRouteIdentity = {
111 threadId: ThreadId.make("thread:shared-runtime:first"),
112 runId: RunId.make("run:shared-runtime:first"),
113 attemptId: RunAttemptId.make("attempt:shared-runtime:first"),
114 providerThreadId: ProviderThreadId.make("provider-thread:shared-runtime:first"),
115 };
116 const second: RunExecutionService.ProviderEventRouteIdentity = {
117 threadId: ThreadId.make("thread:shared-runtime:second"),
118 runId: RunId.make("run:shared-runtime:second"),
119 attemptId: RunAttemptId.make("attempt:shared-runtime:second"),
120 providerThreadId: ProviderThreadId.make("provider-thread:shared-runtime:second"),
121 };
122 const firstTurnId = ProviderTurnId.make("provider-turn:shared-runtime:first");
123 const turnEvent: ProviderAdapter.ProviderAdapterV2Event = {
124 type: "provider_turn.updated",
125 driver,
126 threadId: first.threadId,
127 providerTurn: {
128 id: firstTurnId,
129 providerThreadId: first.providerThreadId,
130 nodeId: NodeId.make("node:shared-runtime:first"),
131 runAttemptId: first.attemptId,
132 nativeTurnRef: null,
133 ordinal: 1,
134 status: "running",
135 startedAt: now,
136 completedAt: null,
137 },
138 };
139 const messageEvent: ProviderAdapter.ProviderAdapterV2Event = {
140 type: "message.updated",
141 driver,
142 message: {
143 createdBy: "agent",
144 creationSource: "provider",
145 id: MessageId.make("message:shared-runtime:first"),
146 threadId: first.threadId,
147 runId: first.runId,
148 nodeId: NodeId.make("node:shared-runtime:first"),
149 role: "assistant",
150 text: "first only",
151 attachments: [],
152 streaming: false,
153 createdAt: now,
154 updatedAt: now,
155 },
156 };
157 const terminalEvent: ProviderAdapter.ProviderAdapterV2Event = {
158 type: "turn.terminal",
159 driver,
160 providerThreadId: first.providerThreadId,
161 providerTurnId: firstTurnId,
162 runOrdinal: 1,
163 status: "completed",
164 failure: null,
165 threadDisposition: "reusable",
166 };
167
168 const firstInitial = RunExecutionService.makeProviderEventRoutingState({
169 identity: first,
170 providerTurnId: null,
171 });
172 const secondInitial = RunExecutionService.makeProviderEventRoutingState({
173 identity: second,
174 providerTurnId: null,
175 });
176 const [firstTurnAccepted, firstAfterTurn] = RunExecutionService.routeProviderEvent(
177 turnEvent,
178 first,
179 firstInitial,
180 );
181 const [secondTurnAccepted, secondAfterTurn] = RunExecutionService.routeProviderEvent(
182 turnEvent,
183 second,
184 secondInitial,
185 );
186
187 assert.isTrue(firstTurnAccepted);
188 assert.isFalse(secondTurnAccepted);
189 assert.isTrue(RunExecutionService.routeProviderEvent(messageEvent, first, firstAfterTurn)[0]);
190 assert.isFalse(
191 RunExecutionService.routeProviderEvent(messageEvent, second, secondAfterTurn)[0],
192 );
193 assert.isTrue(RunExecutionService.routeProviderEvent(terminalEvent, first, firstAfterTurn)[0]);
194 assert.isFalse(
195 RunExecutionService.routeProviderEvent(terminalEvent, second, secondAfterTurn)[0],
196 );
197 }),
198 );
199
200 it("routes only enrolled workflow members after the root turn ends", () => {
201 const threadId = ThreadId.make("thread:late-child");
202 const rootProviderTurnId = ProviderTurnId.make("provider-turn:late-child");
203 const identity: RunExecutionService.ProviderEventRouteIdentity = {
204 threadId,
205 runId: RunId.make("run:late-child"),
206 attemptId: RunAttemptId.make("attempt:late-child"),
207 providerThreadId: ProviderThreadId.make("provider-thread:late-child"),
208 };
209 const childCreated = (
210 childThreadId: ThreadId,
211 parentThreadId = threadId,
212 ): ProviderAdapter.ProviderAdapterV2Event =>
213 ({
214 type: "app_thread.created",
215 driver,
216 appThread: {
217 id: childThreadId,
218 lineage: {
219 parentThreadId,
220 relationshipToParent: "subagent",
221 rootThreadId: threadId,
222 },
223 },
224 }) as ProviderAdapter.ProviderAdapterV2Event;
225 const earlyChild = ThreadId.make("thread:late-child:early");
226 const lateChild = ThreadId.make("thread:late-child:late");
227
228 const initial = RunExecutionService.makeProviderEventRoutingState({
229 identity,
230 providerTurnId: rootProviderTurnId,
231 });
232 const [earlyAccepted, live] = RunExecutionService.routeProviderEvent(
233 childCreated(earlyChild),
234 identity,
235 initial,
236 );
237 assert.isTrue(earlyAccepted);
238 const existingDescendant = ThreadId.make("thread:late-child:existing-descendant");
239 const [descendantAccepted, withDescendant] = RunExecutionService.routeProviderEvent(
240 childCreated(existingDescendant, earlyChild),
241 identity,
242 live,
243 );
244 assert.isTrue(descendantAccepted);
245 const [terminalAccepted, ended] = RunExecutionService.routeProviderEvent(
246 {
247 type: "turn.terminal",
248 driver,
249 providerThreadId: identity.providerThreadId,
250 providerTurnId: rootProviderTurnId,
251 runOrdinal: 1,
252 status: "completed",
253 failure: null,
254 threadDisposition: "reusable",
255 },
256 identity,
257 withDescendant,
258 );
259 assert.isTrue(terminalAccepted);
260 assert.isTrue(ended.ownedThreadIds.has(earlyChild));
261 const [lateAccepted, afterLate] = RunExecutionService.routeProviderEvent(
262 childCreated(lateChild),
263 identity,
264 ended,
265 );
266 assert.isFalse(lateAccepted);
267 assert.isFalse(afterLate.ownedThreadIds.has(lateChild));
268
269 const memberId = ThreadId.make("thread:late-child:workflow-member");
270 assert.isFalse(
271 RunExecutionService.routeProviderEvent(childCreated(memberId, earlyChild), identity, ended)[0],
272 );
273 const coordinator: OrchestrationV2Subagent = {
274 id: NodeId.make("node:late-child:workflow"),
275 threadId,
276 runId: identity.runId,
277 parentNodeId: NodeId.make("node:late-child"),
278 origin: "provider_native",
279 createdBy: "agent",
280 driver,
281 providerInstanceId: ProviderInstanceId.make("codex"),
282 providerThreadId: null,
283 childThreadId: earlyChild,
284 nativeTaskRef: null,
285 prompt: "Review the code",
286 title: "Review",
287 model: null,
288 status: "running",
289 result: null,
290 startedAt: null,
291 completedAt: null,
292 updatedAt: DateTime.makeUnsafe(0),
293 workflow: {
294 phases: [],
295 agents: [{ index: 0, label: "Reviewer", state: "running", childThreadId: memberId }],
296 },
297 };
298 const [foreignAccepted, afterForeign] = RunExecutionService.routeProviderEvent(
299 {
300 type: "subagent.updated",
301 driver,
302 subagent: {
303 ...coordinator,
304 threadId: earlyChild,
305 childThreadId: existingDescendant,
306 runId: RunId.make("run:late-child:new"),
307 },
308 },
309 identity,
310 ended,
311 );
312 assert.isTrue(foreignAccepted);
313 assert.isFalse(
314 RunExecutionService.routeProviderEvent(
315 childCreated(memberId, existingDescendant),
316 identity,
317 afterForeign,
318 )[0],
319 );
320 const [workflowAccepted, afterWorkflow] = RunExecutionService.routeProviderEvent(
321 { type: "subagent.updated", driver, subagent: coordinator },
322 identity,
323 ended,
324 );
325 assert.isTrue(workflowAccepted);
326 assert.isFalse(
327 RunExecutionService.routeProviderEvent(childCreated(memberId), identity, afterWorkflow)[0],
328 );
329 const [memberAccepted, afterMember] = RunExecutionService.routeProviderEvent(
330 childCreated(memberId, earlyChild),
331 identity,
332 afterWorkflow,
333 );
334 assert.isTrue(memberAccepted);
335 assert.isTrue(afterMember.ownedThreadIds.has(memberId));
336 const [unrelatedAccepted] = RunExecutionService.routeProviderEvent(
337 childCreated(ThreadId.make("thread:late-child:unrelated-member"), lateChild),
338 identity,
339 ended,
340 );
341 assert.isFalse(unrelatedAccepted);
342 const [unrelatedDescendantAccepted, afterUnrelatedDescendant] =
343 RunExecutionService.routeProviderEvent(
344 childCreated(ThreadId.make("thread:late-child:unrelated-descendant"), earlyChild),
345 identity,
346 afterWorkflow,
347 );
348 assert.isFalse(unrelatedDescendantAccepted);
349 assert.isFalse(
350 afterUnrelatedDescendant.ownedThreadIds.has(
351 ThreadId.make("thread:late-child:unrelated-descendant"),
352 ),
353 );
354 });
355
356 it("moves a child thread to the run that shows its subagent now", () => {
357 const threadId = ThreadId.make("thread:handoff");
358 const rootProviderTurnId = ProviderTurnId.make("provider-turn:handoff");
359 const identity: RunExecutionService.ProviderEventRouteIdentity = {
360 threadId,
361 runId: RunId.make("run:handoff:1"),
362 attemptId: RunAttemptId.make("attempt:handoff:1"),
363 providerThreadId: ProviderThreadId.make("provider-thread:handoff"),
364 };
365 const resumedChild = ThreadId.make("thread:handoff:resumed");
366 const siblingChild = ThreadId.make("thread:handoff:sibling");
367 const providerThreadOf = (childThreadId: ThreadId) =>
368 ProviderThreadId.make(`provider-thread:${childThreadId}`);
369 const subagent = (childThreadId: ThreadId, runId: RunId): OrchestrationV2Subagent => ({
370 id: NodeId.make(`node:${childThreadId}`),
371 threadId,
372 runId,
373 parentNodeId: NodeId.make("node:handoff"),
374 origin: "provider_native",
375 createdBy: "agent",
376 driver,
377 providerInstanceId: ProviderInstanceId.make("codex"),
378 providerThreadId: providerThreadOf(childThreadId),
379 childThreadId,
380 nativeTaskRef: null,
381 prompt: "Work",
382 title: "Work",
383 model: null,
384 status: "running",
385 result: null,
386 startedAt: null,
387 completedAt: null,
388 updatedAt: DateTime.makeUnsafe(0),
389 });
390 const childMessage = (childThreadId: ThreadId) =>
391 ({
392 type: "message.updated",
393 driver,
394 message: {
395 id: MessageId.make(`message:${childThreadId}`),
396 threadId: childThreadId,
397 runId: null,
398 },
399 }) as ProviderAdapter.ProviderAdapterV2Event;
400 const childTurn = (childThreadId: ThreadId, ordinal: number) =>
401 ({
402 type: "provider_turn.updated",
403 driver,
404 providerTurn: {
405 id: ProviderTurnId.make(`provider-turn:${childThreadId}:${ordinal}`),
406 providerThreadId: providerThreadOf(childThreadId),
407 runAttemptId: null,
408 status: "running",
409 },
410 }) as ProviderAdapter.ProviderAdapterV2Event;
411 const ended = RunExecutionService.routeProviderEvent(
412 {
413 type: "turn.terminal",
414 driver,
415 providerThreadId: identity.providerThreadId,
416 providerTurnId: rootProviderTurnId,
417 runOrdinal: 1,
418 status: "completed",
419 failure: null,
420 threadDisposition: "reusable",
421 },
422 identity,
423 RunExecutionService.makeProviderEventRoutingState({
424 identity,
425 providerTurnId: rootProviderTurnId,
426 relatedThreadIds: [resumedChild, siblingChild],
427 relatedProviderThreadIds: [providerThreadOf(resumedChild), providerThreadOf(siblingChild)],
428 }),
429 )[1];
430 const [firstTurnAccepted, withChildTurn] = RunExecutionService.routeProviderEvent(
431 childTurn(resumedChild, 1),
432 identity,
433 ended,
434 );
435 assert.isTrue(firstTurnAccepted);
436 const [movedAccepted, afterMove] = RunExecutionService.routeProviderEvent(
437 {
438 type: "subagent.updated",
439 driver,
440 subagent: subagent(resumedChild, RunId.make("run:handoff:2")),
441 },
442 identity,
443 withChildTurn,
444 );
445 assert.isFalse(movedAccepted);
446 assert.isFalse(
447 RunExecutionService.routeProviderEvent(childMessage(resumedChild), identity, afterMove)[0],
448 );
449 assert.isTrue(
450 RunExecutionService.routeProviderEvent(childMessage(siblingChild), identity, afterMove)[0],
451 );
452 assert.isFalse(
453 RunExecutionService.routeProviderEvent(childTurn(resumedChild, 1), identity, afterMove)[0],
454 );
455 assert.isFalse(
456 RunExecutionService.routeProviderEvent(childTurn(resumedChild, 2), identity, afterMove)[0],
457 );
458 assert.isTrue(
459 RunExecutionService.routeProviderEvent(childTurn(siblingChild, 1), identity, afterMove)[0],
460 );
461 assert.isTrue(
462 RunExecutionService.routeProviderEvent(
463 { type: "subagent.updated", driver, subagent: subagent(siblingChild, identity.runId) },
464 identity,
465 afterMove,
466 )[0],
467 );
468 const resuming: RunExecutionService.ProviderEventRouteIdentity = {
469 ...identity,
470 runId: RunId.make("run:handoff:2"),
471 attemptId: RunAttemptId.make("attempt:handoff:2"),
472 };
473 const resumingState = RunExecutionService.makeProviderEventRoutingState({
474 identity: resuming,
475 providerTurnId: ProviderTurnId.make("provider-turn:handoff:2"),
476 });
477 assert.isFalse(
478 RunExecutionService.routeProviderEvent(childMessage(resumedChild), resuming, resumingState)[0],
479 );
480 const [moveAccepted, afterTake] = RunExecutionService.routeProviderEvent(
481 { type: "subagent.updated", driver, subagent: subagent(resumedChild, resuming.runId) },
482 resuming,
483 resumingState,
484 );
485 assert.isTrue(moveAccepted);
486 assert.isTrue(
487 RunExecutionService.routeProviderEvent(childMessage(resumedChild), resuming, afterTake)[0],
488 );
489 });
490
491 it("does not route a superseded attempt through a reused provider thread", () => {
492 const threadId = ThreadId.make("thread:shared-runtime:restart");
493 const providerThreadId = ProviderThreadId.make("provider-thread:shared-runtime:restart");
494 const oldAttempt: RunExecutionService.ProviderEventRouteIdentity = {
495 threadId,
496 runId: RunId.make("run:shared-runtime:restart"),
497 attemptId: RunAttemptId.make("attempt:shared-runtime:restart:old"),
498 providerThreadId,
499 };
500 const newAttempt: RunExecutionService.ProviderEventRouteIdentity = {
501 ...oldAttempt,
502 attemptId: RunAttemptId.make("attempt:shared-runtime:restart:new"),
503 };
504 const oldTurnEvent: ProviderAdapter.ProviderAdapterV2Event = {
505 type: "provider_turn.updated",
506 driver,
507 threadId,
508 providerTurn: {
509 id: ProviderTurnId.make("provider-turn:shared-runtime:restart:old"),
510 providerThreadId,
511 nodeId: NodeId.make("node:shared-runtime:restart:old"),
512 runAttemptId: oldAttempt.attemptId,
513 nativeTurnRef: null,
514 ordinal: 1,
515 status: "interrupted",
516 startedAt: null,
517 completedAt: null,
518 },
519 };
520
521 const newState = RunExecutionService.makeProviderEventRoutingState({
522 identity: newAttempt,
523 providerTurnId: null,
524 });
525 assert.isFalse(RunExecutionService.routeProviderEvent(oldTurnEvent, newAttempt, newState)[0]);
526 });
527
528 it("routes only exact same-thread background items inherited from settled runs", () => {
529 const threadId = ThreadId.make("thread:inherited-background-routing");
530 const otherThreadId = ThreadId.make("thread:inherited-background-routing:other");
531 const priorRunId = RunId.make("run:inherited-background-routing:prior");
532 const currentRunId = RunId.make("run:inherited-background-routing:current");
533 const itemId = TurnItemId.make("turn-item:inherited-background-routing");
534 const identity: RunExecutionService.ProviderEventRouteIdentity = {
535 threadId,
536 runId: currentRunId,
537 attemptId: RunAttemptId.make("attempt:inherited-background-routing:current"),
538 providerThreadId: ProviderThreadId.make("provider-thread:inherited-background-routing:current"),
539 };
540 const initial = RunExecutionService.makeProviderEventRoutingState({
541 identity,
542 inheritedBackgroundTurnItems: [{ id: itemId, runId: priorRunId }],
543 providerTurnId: null,
544 });
545 const inheritedRunning = {
546 type: "turn_item.updated",
547 driver,
548 turnItem: {
549 id: itemId,
550 threadId,
551 runId: priorRunId,
552 providerTurnId: null,
553 ordinal: 1,
554 type: "subagent",
555 status: "running",
556 },
557 } as Extract<ProviderAdapter.ProviderAdapterV2Event, { type: "turn_item.updated" }>;
558 const unrelatedRunItem = {
559 ...inheritedRunning,
560 turnItem: {
561 ...inheritedRunning.turnItem,
562 runId: RunId.make("run:inherited-background-routing:unrelated"),
563 },
564 } as ProviderAdapter.ProviderAdapterV2Event;
565 const unlistedPriorItem = {
566 ...inheritedRunning,
567 turnItem: {
568 ...inheritedRunning.turnItem,
569 id: TurnItemId.make("turn-item:inherited-background-routing:unrelated"),
570 },
571 } as ProviderAdapter.ProviderAdapterV2Event;
572 const unrelatedThreadItem = {
573 ...inheritedRunning,
574 turnItem: { ...inheritedRunning.turnItem, threadId: otherThreadId },
575 } as ProviderAdapter.ProviderAdapterV2Event;
576 const ordinaryItem = {
577 ...inheritedRunning,
578 turnItem: { ...inheritedRunning.turnItem, type: "reasoning" as const },
579 } as ProviderAdapter.ProviderAdapterV2Event;
580 const inheritedTerminal = {
581 ...inheritedRunning,
582 turnItem: { ...inheritedRunning.turnItem, status: "completed" as const },
583 } as ProviderAdapter.ProviderAdapterV2Event;
584
585 const [runningAccepted, afterRunning] = RunExecutionService.routeProviderEvent(
586 inheritedRunning,
587 identity,
588 initial,
589 );
590 assert.isTrue(runningAccepted);
591 assert.isFalse(
592 RunExecutionService.routeProviderEvent(unrelatedRunItem, identity, afterRunning)[0],
593 );
594 assert.isFalse(
595 RunExecutionService.routeProviderEvent(unlistedPriorItem, identity, afterRunning)[0],
596 );
597 assert.isFalse(
598 RunExecutionService.routeProviderEvent(unrelatedThreadItem, identity, afterRunning)[0],
599 );
600 assert.isFalse(RunExecutionService.routeProviderEvent(ordinaryItem, identity, afterRunning)[0]);
601
602 const [terminalAccepted, afterTerminal] = RunExecutionService.routeProviderEvent(
603 inheritedTerminal,
604 identity,
605 afterRunning,
606 );
607 assert.isTrue(terminalAccepted);
608 assert.isFalse(
609 RunExecutionService.routeProviderEvent(inheritedRunning, identity, afterTerminal)[0],
610 "a nonterminal replay must not resurrect an inherited terminal",
611 );
612 });
613
614 it("routes an earlier run's plan rows to this thread's active root turn only", () => {
615 const threadId = ThreadId.make("thread:earlier-plan-routing");
616 const priorRunId = RunId.make("run:earlier-plan-routing:prior");
617 const rootTurnId = ProviderTurnId.make("provider-turn:earlier-plan-routing:current");
618 const identity: RunExecutionService.ProviderEventRouteIdentity = {
619 threadId,
620 runId: RunId.make("run:earlier-plan-routing:current"),
621 attemptId: RunAttemptId.make("attempt:earlier-plan-routing:current"),
622 providerThreadId: ProviderThreadId.make("provider-thread:earlier-plan-routing"),
623 };
624 const active = RunExecutionService.makeProviderEventRoutingState({
625 identity,
626 providerTurnId: rootTurnId,
627 });
628 const planRow = {
629 type: "turn_item.updated",
630 driver,
631 turnItem: {
632 id: TurnItemId.make("turn-item:earlier-plan-routing"),
633 threadId,
634 runId: priorRunId,
635 type: "todo_list",
636 },
637 } as ProviderAdapter.ProviderAdapterV2Event;
638 const plan = {
639 type: "plan.updated",
640 driver,
641 plan: { threadId, runId: priorRunId },
642 } as ProviderAdapter.ProviderAdapterV2Event;
643 const otherThreadPlan = {
644 type: "plan.updated",
645 driver,
646 plan: { threadId: ThreadId.make("thread:earlier-plan-routing:other"), runId: priorRunId },
647 } as ProviderAdapter.ProviderAdapterV2Event;
648 const ordinaryRow = {
649 type: "turn_item.updated",
650 driver,
651 turnItem: {
652 id: TurnItemId.make("turn-item:earlier-plan-routing:tool"),
653 threadId,
654 runId: priorRunId,
655 type: "reasoning",
656 },
657 } as ProviderAdapter.ProviderAdapterV2Event;
658 const [, ended] = RunExecutionService.routeProviderEvent(
659 {
660 type: "turn.terminal",
661 driver,
662 providerThreadId: identity.providerThreadId,
663 providerTurnId: rootTurnId,
664 runOrdinal: 2,
665 status: "completed",
666 failure: null,
667 threadDisposition: "reusable",
668 },
669 identity,
670 active,
671 );
672
673 assert.isTrue(RunExecutionService.routeProviderEvent(planRow, identity, active)[0]);
674 assert.isTrue(RunExecutionService.routeProviderEvent(plan, identity, active)[0]);
675 assert.isFalse(RunExecutionService.routeProviderEvent(otherThreadPlan, identity, active)[0]);
676 assert.isFalse(RunExecutionService.routeProviderEvent(ordinaryRow, identity, active)[0]);
677 assert.isFalse(RunExecutionService.routeProviderEvent(planRow, identity, ended)[0]);
678 assert.isFalse(RunExecutionService.routeProviderEvent(plan, identity, ended)[0]);
679 });
680
681 it("selects only live background items from non-completed settled prior runs", () => {
682 const threadId = ThreadId.make("thread:inherited-background-selection");
683 const otherThreadId = ThreadId.make("thread:inherited-background-selection:other");
684 const currentProviderThreadId = ProviderThreadId.make(
685 "provider-thread:inherited-background-selection:current",
686 );
687 const foreignProviderThreadId = ProviderThreadId.make(
688 "provider-thread:inherited-background-selection:foreign-session",
689 );
690 const interruptedRunId = RunId.make("run:inherited-background-selection:interrupted");
691 const failedRunId = RunId.make("run:inherited-background-selection:failed");
692 const cancelledRunId = RunId.make("run:inherited-background-selection:cancelled");
693 const completedRunId = RunId.make("run:inherited-background-selection:completed");
694 const rolledBackRunId = RunId.make("run:inherited-background-selection:rolled-back");
695 const currentRunId = RunId.make("run:inherited-background-selection:current");
696 const inheritedItemId = TurnItemId.make("turn-item:inherited-background-selection:live");
697 const failedItemId = TurnItemId.make("turn-item:inherited-background-selection:failed");
698 const cancelledItemId = TurnItemId.make("turn-item:inherited-background-selection:cancelled");
699 const makeRun = (
700 id: RunId,
701 ordinal: number,
702 status: OrchestrationV2Run["status"],
703 runThreadId = threadId,
704 ) =>
705 ({
706 id,
707 threadId: runThreadId,
708 ordinal,
709 status,
710 }) as OrchestrationV2Run;
711 const makeItem = (
712 id: TurnItemId,
713 runId: RunId,
714 status: OrchestrationV2TurnItem["status"],
715 type: OrchestrationV2TurnItem["type"] = "subagent",
716 itemThreadId = threadId,
717 providerThreadId = currentProviderThreadId,
718 ) =>
719 ({
720 id,
721 threadId: itemThreadId,
722 runId,
723 providerThreadId,
724 type,
725 status,
726 }) as OrchestrationV2TurnItem;
727
728 const selected = RunExecutionService.selectInheritedBackgroundTurnItems({
729 threadId,
730 currentProviderThreadId,
731 currentRunOrdinal: 6,
732 runs: [
733 makeRun(interruptedRunId, 1, "interrupted"),
734 makeRun(failedRunId, 2, "failed"),
735 makeRun(cancelledRunId, 3, "cancelled"),
736 makeRun(completedRunId, 4, "completed"),
737 makeRun(rolledBackRunId, 5, "rolled_back"),
738 makeRun(currentRunId, 6, "running"),
739 makeRun(
740 RunId.make("run:inherited-background-selection:other"),
741 1,
742 "interrupted",
743 otherThreadId,
744 ),
745 ],
746 turnItems: [
747 makeItem(inheritedItemId, interruptedRunId, "running"),
748 makeItem(failedItemId, failedRunId, "running"),
749 makeItem(cancelledItemId, cancelledRunId, "running"),
750 makeItem(
751 TurnItemId.make("turn-item:inherited-background-selection:completed-run"),
752 completedRunId,
753 "running",
754 ),
755 makeItem(
756 TurnItemId.make("turn-item:inherited-background-selection:terminal"),
757 interruptedRunId,
758 "completed",
759 ),
760 makeItem(
761 TurnItemId.make("turn-item:inherited-background-selection:ordinary"),
762 interruptedRunId,
763 "running",
764 "reasoning",
765 ),
766 makeItem(
767 TurnItemId.make("turn-item:inherited-background-selection:rolled-back"),
768 rolledBackRunId,
769 "running",
770 ),
771 makeItem(
772 TurnItemId.make("turn-item:inherited-background-selection:other-thread"),
773 interruptedRunId,
774 "running",
775 "subagent",
776 otherThreadId,
777 ),
778 makeItem(
779 TurnItemId.make("turn-item:inherited-background-selection:foreign-provider"),
780 interruptedRunId,
781 "running",
782 "subagent",
783 threadId,
784 foreignProviderThreadId,
785 ),
786 ],
787 });
788
789 assert.deepEqual(selected, [
790 { id: inheritedItemId, runId: interruptedRunId },
791 { id: failedItemId, runId: failedRunId },
792 { id: cancelledItemId, runId: cancelledRunId },
793 ]);
794 });
795
796 it("does not carry interrupted or still-running child ownership into later attempts", () => {
797 assert.isFalse(RunExecutionService.canRouteRelatedSubagent("interrupted"));
798 assert.isFalse(RunExecutionService.canRouteRelatedSubagent("failed"));
799 assert.isFalse(RunExecutionService.canRouteRelatedSubagent("cancelled"));
800 assert.isTrue(RunExecutionService.canRouteRelatedSubagent("completed"));
801 assert.isFalse(RunExecutionService.canRouteRelatedSubagent("running"));
802
803 const threadId = ThreadId.make("thread:related-child:next-attempt");
804 const childThreadId = ThreadId.make("thread:related-child:interrupted");
805 const identity: RunExecutionService.ProviderEventRouteIdentity = {
806 threadId,
807 runId: RunId.make("run:related-child:next-attempt"),
808 attemptId: RunAttemptId.make("attempt:related-child:next-attempt"),
809 providerThreadId: ProviderThreadId.make("provider-thread:related-child:next-attempt"),
810 };
811 const state = RunExecutionService.makeProviderEventRoutingState({
812 identity,
813 providerTurnId: null,
814 relatedThreadIds: RunExecutionService.canRouteRelatedSubagent("interrupted")
815 ? [childThreadId]
816 : [],
817 });
818 const childNodeId = NodeId.make("node:related-child:interrupted");
819 const lateChildNode = {
820 type: "node.updated",
821 driver,
822 node: {
823 id: childNodeId,
824 threadId: childThreadId,
825 runId: null,
826 parentNodeId: null,
827 rootNodeId: childNodeId,
828 kind: "root_turn",
829 status: "completed",
830 countsForRun: false,
831 providerThreadId: null,
832 providerTurnId: null,
833 nativeItemRef: null,
834 runtimeRequestId: null,
835 checkpointScopeId: null,
836 startedAt: null,
837 completedAt: null,
838 },
839 } satisfies ProviderAdapter.ProviderAdapterV2Event;
840
841 assert.isFalse(RunExecutionService.routeProviderEvent(lateChildNode, identity, state)[0]);
842 });
843
844 it.effect("rechecks run ownership immediately before calling the provider", () =>
845 Effect.gen(function* () {
846 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
847 const guardCalls = yield* Ref.make(0);
848 const providerStarts = yield* Ref.make(0);
849 const threadId = ThreadId.make("thread:run-execution-start-guard");
850 const runId = RunId.make("run:run-execution-start-guard");
851 const attemptId = RunAttemptId.make("attempt:run-execution-start-guard");
852 const providerThreadId = ProviderThreadId.make("provider-thread:run-execution-start-guard");
853 const providerInstanceId = ProviderInstanceId.make("codex");
854 const providerSessionId = ProviderSessionId.make("session:run-execution-start-guard");
855 const rootNodeId = NodeId.make("node:run-execution-start-guard");
856 const run = {
857 id: runId,
858 threadId,
859 ordinal: 1,
860 providerInstanceId,
861 } as OrchestrationV2Run;
862 const rootNode = { id: rootNodeId } as OrchestrationV2ExecutionNode;
863 const providerThread = {
864 id: providerThreadId,
865 driver,
866 } as OrchestrationV2ProviderThread;
867 const attempt = {
868 id: attemptId,
869 providerTurnId: null,
870 } as OrchestrationV2RunAttempt;
871 const session = {
872 events: Stream.never,
873 startTurn: () => Ref.update(providerStarts, (count) => count + 1),
874 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime;
875
876 yield* runExecution.startRootRun({
877 commandId: CommandId.make("command:run-execution-start-guard"),
878 appThread: rootAppThread(threadId),
879 providerSessionId,
880 session,
881 run,
882 rootNode,
883 checkpointScope: {
884 id: CheckpointScopeId.make("checkpoint-scope:run-execution-start-guard"),
885 } as OrchestrationV2CheckpointScope,
886 providerThread,
887 attempt,
888 attemptId,
889 providerTurnOrdinal: 1,
890 shouldStartProviderTurn: () =>
891 Ref.modify(guardCalls, (calls) => [calls === 0, calls + 1] as const),
892 message: {
893 messageId: MessageId.make("message:run-execution-start-guard"),
894 text: "Do not start after ownership changes.",
895 attachments: [],
896 createdBy: "user",
897 creationSource: "web",
898 },
899 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
900 runtimePolicy: {
901 runtimeMode: "full-access",
902 interactionMode: "default",
903 cwd: process.cwd(),
904 approvalPolicy: "never",
905 sandboxPolicy: {
906 type: "readOnly",
907 access: { type: "fullAccess" },
908 networkAccess: false,
909 },
910 },
911 });
912
913 assert.equal(yield* Ref.get(guardCalls), 2);
914 assert.equal(yield* Ref.get(providerStarts), 0);
915 }).pipe(Effect.provide(layerRunExecutionTest)),
916 );
917
918 it.effect("passes the thread's MCP app context to the provider under a safe key", () =>
919 Effect.gen(function* () {
920 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
921 const started = yield* Deferred.make<ProviderAdapter.ProviderAdapterV2TurnInput>();
922 const threadId = ThreadId.make("thread:run-execution-app-context");
923 const providerInstanceId = ProviderInstanceId.make("codex");
924 const attemptId = RunAttemptId.make("attempt:run-execution-app-context");
925 const session = {
926 events: Stream.never,
927 startTurn: (input: ProviderAdapter.ProviderAdapterV2TurnInput) =>
928 Deferred.succeed(started, input),
929 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime;
930
931 yield* runExecution.startRootRun({
932 commandId: CommandId.make("command:run-execution-app-context"),
933 appThread: rootAppThread(threadId),
934 providerSessionId: ProviderSessionId.make("session:run-execution-app-context"),
935 session,
936 run: {
937 id: RunId.make("run:run-execution-app-context"),
938 threadId,
939 ordinal: 1,
940 providerInstanceId,
941 } as OrchestrationV2Run,
942 rootNode: {
943 id: NodeId.make("node:run-execution-app-context"),
944 } as OrchestrationV2ExecutionNode,
945 checkpointScope: {
946 id: CheckpointScopeId.make("checkpoint-scope:run-execution-app-context"),
947 } as OrchestrationV2CheckpointScope,
948 providerThread: {
949 id: ProviderThreadId.make("provider-thread:run-execution-app-context"),
950 driver,
951 } as OrchestrationV2ProviderThread,
952 attempt: { id: attemptId, providerTurnId: null } as OrchestrationV2RunAttempt,
953 attemptId,
954 providerTurnOrdinal: 1,
955 message: {
956 messageId: MessageId.make("message:run-execution-app-context"),
957 text: "What is on my list?",
958 attachments: [],
959 createdBy: "user",
960 creationSource: "web",
961 },
962 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
963 runtimePolicy: { runtimeMode: "full-access", interactionMode: "default", cwd: process.cwd() },
964 });
965
966 const input = yield* Deferred.await(started);
967 assert.deepEqual(input.appContext, [
968 { key: "mcp_app_turn-item_provider_codex_native-item_call-1", text: "2 overdue" },
969 ]);
970 }).pipe(
971 Effect.provide(
972 RunExecutionService.layer.pipe(
973 Layer.provide(
974 Layer.mergeAll(
975 PreviewAgentActivity.layer,
976 Layer.mock(McpAppModelContext.McpAppModelContext)({
977 forThread: () =>
978 Effect.succeed([
979 {
980 itemId: "turn-item:provider:codex:native-item:call-1",
981 server: "my tools>",
982 tool: "list",
983 text: "2 overdue",
984 },
985 ]),
986 }),
987 Layer.mock(CheckpointService.CheckpointServiceV2)({
988 captureBaseline: () => Effect.void,
989 }),
990 layerTurnItemPositionsTest,
991 mockEventSink({}),
992 IdAllocator.layer,
993 ThreadCommandExecutor.layer,
994 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
995 ingestNormalized: () => Effect.succeed([]),
996 }),
997 ServerSettings.layerTest(),
998 ),
999 ),
1000 ),
1001 ),
1002 ),
1003 );
1004
1005 it.effect("fails the run when its ownership check cannot be read before calling the provider", () =>
1006 Effect.gen(function* () {
1007 const guardCalls = yield* Ref.make(0);
1008 const providerStarts = yield* Ref.make(0);
1009 const writes = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);
1010 const threadId = ThreadId.make("thread:run-execution-start-guard-read");
1011 const runId = RunId.make("run:run-execution-start-guard-read");
1012 const attemptId = RunAttemptId.make("attempt:run-execution-start-guard-read");
1013 const providerInstanceId = ProviderInstanceId.make("codex");
1014 const layerTest = RunExecutionService.layer.pipe(
1015 Layer.provide(
1016 Layer.mergeAll(
1017 McpAppModelContext.layerEmpty,
1018 PreviewAgentActivity.layer,
1019 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
1020 layerTurnItemPositionsTest,
1021 mockEventSink({
1022 writeIfRunCurrent: (input) =>
1023 Ref.update(writes, (current) => [...current, ...input.events]).pipe(
1024 Effect.as({ committed: true, storedEvents: [] }),
1025 ),
1026 }),
1027 IdAllocator.layer,
1028 ThreadCommandExecutor.layer,
1029 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
1030 ingestNormalized: () => Effect.succeed([]),
1031 }),
1032 ServerSettings.layerTest(),
1033 ),
1034 ),
1035 );
1036
1037 yield* Effect.gen(function* () {
1038 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
1039 yield* runExecution.startRootRun({
1040 commandId: CommandId.make("command:run-execution-start-guard-read"),
1041 appThread: rootAppThread(threadId),
1042 providerSessionId: ProviderSessionId.make("session:run-execution-start-guard-read"),
1043 session: {
1044 events: Stream.never,
1045 startTurn: () => Ref.update(providerStarts, (count) => count + 1),
1046 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
1047 run: { id: runId, threadId, ordinal: 1, providerInstanceId } as OrchestrationV2Run,
1048 rootNode: {
1049 id: NodeId.make("node:run-execution-start-guard-read"),
1050 } as OrchestrationV2ExecutionNode,
1051 checkpointScope: {
1052 id: CheckpointScopeId.make("checkpoint-scope:run-execution-start-guard-read"),
1053 } as OrchestrationV2CheckpointScope,
1054 providerThread: {
1055 id: ProviderThreadId.make("provider-thread:run-execution-start-guard-read"),
1056 driver,
1057 } as OrchestrationV2ProviderThread,
1058 attempt: { id: attemptId, providerTurnId: null } as OrchestrationV2RunAttempt,
1059 attemptId,
1060 providerTurnOrdinal: 1,
1061 shouldStartProviderTurn: () =>
1062 Ref.getAndUpdate(guardCalls, (calls) => calls + 1).pipe(
1063 Effect.flatMap((calls) =>
1064 calls === 0
1065 ? Effect.succeed(true)
1066 : Effect.fail(
1067 new ProjectionStore.ProjectionStoreReadError({
1068 threadId,
1069 cause: "database unavailable",
1070 }),
1071 ),
1072 ),
1073 ),
1074 shouldFinalizeRun: () =>
1075 Effect.fail(
1076 new ProjectionStore.ProjectionStoreReadError({
1077 threadId,
1078 cause: "database unavailable",
1079 }),
1080 ),
1081 message: {
1082 messageId: MessageId.make("message:run-execution-start-guard-read"),
1083 text: "Start while the store is down.",
1084 attachments: [],
1085 createdBy: "user",
1086 creationSource: "web",
1087 },
1088 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
1089 runtimePolicy: {
1090 runtimeMode: "full-access",
1091 interactionMode: "default",
1092 cwd: process.cwd(),
1093 approvalPolicy: "never",
1094 sandboxPolicy: {
1095 type: "readOnly",
1096 access: { type: "fullAccess" },
1097 networkAccess: false,
1098 },
1099 },
1100 });
1101 }).pipe(Effect.provide(layerTest));
1102
1103 assert.equal(yield* Ref.get(guardCalls), 2);
1104 assert.equal(yield* Ref.get(providerStarts), 0);
1105 const runUpdate = (yield* Ref.get(writes)).find((event) => event.type === "run.updated");
1106 assert.equal(
1107 runUpdate?.type === "run.updated" ? runUpdate.payload.status : undefined,
1108 "failed",
1109 );
1110 }),
1111 );
1112
1113 it.effect(
1114 "dispatches only attachment-free compact commands through the native compaction path",
1115 () =>
1116 Effect.gen(function* () {
1117 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
1118 const calls: Array<string> = [];
1119 const attachment = ChatFileAttachment.make({
1120 type: "file",
1121 id: ChatAttachmentId.make("compact-file-12345678-1234-1234-1234-123456789abc"),
1122 name: "notes.txt",
1123 mimeType: "text/plain",
1124 sizeBytes: 1,
1125 });
1126 const cases = [
1127 { text: " /compact ", attachments: [], expected: "compact" },
1128 { text: "/compact", attachments: [attachment], expected: "prompt" },
1129 { text: "Continue the work", attachments: [], expected: "prompt" },
1130 ];
1131 for (const [index, testCase] of cases.entries()) {
1132 const threadId = ThreadId.make(`thread:compact-routing:${index}`);
1133 const attemptId = RunAttemptId.make(`attempt:compact-routing:${index}`);
1134 yield* runExecution.startRootRun({
1135 commandId: CommandId.make(`command:compact-routing:${index}`),
1136 appThread: rootAppThread(threadId),
1137 providerSessionId: ProviderSessionId.make(`session:compact-routing:${index}`),
1138 session: {
1139 events: Stream.never,
1140 startTurn: () =>
1141 Effect.sync(() => {
1142 calls.push("prompt");
1143 }),
1144 compactThread: () =>
1145 Effect.sync(() => {
1146 calls.push("compact");
1147 }),
1148 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
1149 run: {
1150 id: RunId.make(`run:compact-routing:${index}`),
1151 threadId,
1152 ordinal: 1,
1153 providerInstanceId: ProviderInstanceId.make("codex"),
1154 } as OrchestrationV2Run,
1155 rootNode: {
1156 id: NodeId.make(`node:compact-routing:${index}`),
1157 } as OrchestrationV2ExecutionNode,
1158 checkpointScope: {
1159 id: CheckpointScopeId.make(`checkpoint:compact-routing:${index}`),
1160 } as OrchestrationV2CheckpointScope,
1161 providerThread: {
1162 id: ProviderThreadId.make(`provider-thread:compact-routing:${index}`),
1163 driver,
1164 } as OrchestrationV2ProviderThread,
1165 attempt: { id: attemptId, providerTurnId: null } as OrchestrationV2RunAttempt,
1166 attemptId,
1167 providerTurnOrdinal: 1,
1168 message: {
1169 messageId: MessageId.make(`message:compact-routing:${index}`),
1170 text: testCase.text,
1171 attachments: testCase.attachments,
1172 createdBy: "user",
1173 creationSource: "web",
1174 },
1175 modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" },
1176 runtimePolicy: {
1177 runtimeMode: "full-access",
1178 interactionMode: "default",
1179 cwd: process.cwd(),
1180 approvalPolicy: "never",
1181 sandboxPolicy: {
1182 type: "readOnly",
1183 access: { type: "fullAccess" },
1184 networkAccess: false,
1185 },
1186 },
1187 });
1188 }
1189 assert.deepEqual(
1190 calls,
1191 cases.map((testCase) => testCase.expected),
1192 );
1193 }).pipe(Effect.provide(layerRunExecutionTest)),
1194 );
1195
1196 it.effect("starts the provider when checkpoint baseline capture fails", () =>
1197 Effect.gen(function* () {
1198 const threadId = ThreadId.make("thread:run-execution-baseline-failure");
1199 const runId = RunId.make("run:run-execution-baseline-failure");
1200 const attemptId = RunAttemptId.make("attempt:run-execution-baseline-failure");
1201 const providerThreadId = ProviderThreadId.make(
1202 "provider-thread:run-execution-baseline-failure",
1203 );
1204 const providerInstanceId = ProviderInstanceId.make("codex");
1205 const providerSessionId = ProviderSessionId.make("session:run-execution-baseline-failure");
1206 const rootNodeId = NodeId.make("node:run-execution-baseline-failure");
1207 const checkpointScope = {
1208 id: CheckpointScopeId.make("checkpoint-scope:run-execution-baseline-failure"),
1209 } as OrchestrationV2CheckpointScope;
1210 const providerStarts = yield* Ref.make(0);
1211 const writes = yield* Ref.make<
1212 ReadonlyArray<{
1213 events: ReadonlyArray<OrchestrationV2DomainEvent>;
1214 effects: ReadonlyArray<PendingOrchestrationEffectV2>;
1215 }>
1216 >([]);
1217 const layerTest = RunExecutionService.layer.pipe(
1218 Layer.provide(
1219 Layer.mergeAll(
1220 McpAppModelContext.layerEmpty,
1221 PreviewAgentActivity.layer,
1222 Layer.mock(CheckpointService.CheckpointServiceV2)({
1223 captureBaseline: () =>
1224 Effect.fail(
1225 new CheckpointService.CheckpointBaselineCaptureError({
1226 scopeId: checkpointScope.id,
1227 ordinalWithinScope: 0,
1228 cause: new Error("VCS process timed out"),
1229 }),
1230 ),
1231 }),
1232 layerTurnItemPositionsTest,
1233 mockEventSink({
1234 writeWithEffects: (input) =>
1235 Ref.update(writes, (current) => [
1236 ...current,
1237 { events: input.events, effects: input.effects },
1238 ]).pipe(Effect.as([])),
1239 }),
1240 IdAllocator.layer,
1241 ThreadCommandExecutor.layer,
1242 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
1243 ingestNormalized: () => Effect.succeed([]),
1244 }),
1245 ServerSettings.layerTest(),
1246 ),
1247 ),
1248 );
1249
1250 yield* Effect.gen(function* () {
1251 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
1252 yield* runExecution.startRootRun({
1253 commandId: CommandId.make("command:run-execution-baseline-failure"),
1254 appThread: rootAppThread(threadId),
1255 providerSessionId,
1256 session: {
1257 events: Stream.never,
1258 startTurn: () => Ref.update(providerStarts, (count) => count + 1),
1259 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
1260 run: {
1261 id: runId,
1262 threadId,
1263 ordinal: 1,
1264 providerInstanceId,
1265 } as OrchestrationV2Run,
1266 rootNode: { id: rootNodeId } as OrchestrationV2ExecutionNode,
1267 checkpointScope,
1268 providerThread: {
1269 id: providerThreadId,
1270 driver,
1271 } as OrchestrationV2ProviderThread,
1272 attempt: {
1273 id: attemptId,
1274 providerTurnId: null,
1275 } as OrchestrationV2RunAttempt,
1276 attemptId,
1277 providerTurnOrdinal: 1,
1278 message: {
1279 messageId: MessageId.make("message:run-execution-baseline-failure"),
1280 text: "Start the provider without a baseline.",
1281 attachments: [],
1282 createdBy: "user",
1283 creationSource: "web",
1284 },
1285 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
1286 runtimePolicy: {
1287 runtimeMode: "full-access",
1288 interactionMode: "default",
1289 cwd: process.cwd(),
1290 approvalPolicy: "never",
1291 sandboxPolicy: {
1292 type: "readOnly",
1293 access: { type: "fullAccess" },
1294 networkAccess: false,
1295 },
1296 },
1297 });
1298 }).pipe(Effect.provide(layerTest));
1299
1300 assert.equal(yield* Ref.get(providerStarts), 1);
1301
1302 const events = (yield* Ref.get(writes)).flatMap((write) => [...write.events]);
1303 const failedEvent = events.find(
1304 (event) =>
1305 (event.type === "run.updated" ||
1306 event.type === "run-attempt.updated" ||
1307 event.type === "node.updated") &&
1308 event.payload.status === "failed",
1309 );
1310 assert.isUndefined(failedEvent);
1311 const errorItem = events.find(
1312 (event) => event.type === "turn-item.updated" && event.payload.type === "error",
1313 );
1314 assert.isUndefined(errorItem);
1315 }),
1316 );
1317
1318 it.effect("prompts the provider only after a forked baseline capture finishes", () =>
1319 Effect.gen(function* () {
1320 const threadId = ThreadId.make("thread:run-execution-forked-baseline");
1321 const runId = RunId.make("run:run-execution-forked-baseline");
1322 const attemptId = RunAttemptId.make("attempt:run-execution-forked-baseline");
1323 const providerInstanceId = ProviderInstanceId.make("codex");
1324 const checkpointScope = {
1325 id: CheckpointScopeId.make("checkpoint-scope:run-execution-forked-baseline"),
1326 } as OrchestrationV2CheckpointScope;
1327 const run = { id: runId, threadId, ordinal: 1, providerInstanceId } as OrchestrationV2Run;
1328 const order = yield* Ref.make<ReadonlyArray<string>>([]);
1329 const layerTest = RunExecutionService.layer.pipe(
1330 Layer.provide(
1331 Layer.mergeAll(
1332 McpAppModelContext.layerEmpty,
1333 PreviewAgentActivity.layer,
1334 Layer.mock(CheckpointService.CheckpointServiceV2)({
1335 captureBaseline: () =>
1336 Effect.sleep("1 second").pipe(
1337 Effect.andThen(Ref.update(order, (current) => [...current, "baseline"])),
1338 ),
1339 }),
1340 layerTurnItemPositionsTest,
1341 mockEventSink({}),
1342 IdAllocator.layer,
1343 ThreadCommandExecutor.layer,
1344 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
1345 ingestNormalized: () => Effect.succeed([]),
1346 }),
1347 ServerSettings.layerTest(),
1348 ),
1349 ),
1350 );
1351
1352 yield* Effect.gen(function* () {
1353 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
1354 const baseline = yield* runExecution.forkBaseline({
1355 appThread: rootAppThread(threadId),
1356 run,
1357 checkpointScope,
1358 });
1359 const started = yield* runExecution
1360 .startRootRun({
1361 commandId: CommandId.make("command:run-execution-forked-baseline"),
1362 appThread: rootAppThread(threadId),
1363 providerSessionId: ProviderSessionId.make("session:run-execution-forked-baseline"),
1364 session: {
1365 events: Stream.never,
1366 startTurn: () => Ref.update(order, (current) => [...current, "prompt"]),
1367 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
1368 run,
1369 rootNode: {
1370 id: NodeId.make("node:run-execution-forked-baseline"),
1371 } as OrchestrationV2ExecutionNode,
1372 checkpointScope,
1373 providerThread: {
1374 id: ProviderThreadId.make("provider-thread:run-execution-forked-baseline"),
1375 driver,
1376 } as OrchestrationV2ProviderThread,
1377 attempt: { id: attemptId, providerTurnId: null } as OrchestrationV2RunAttempt,
1378 attemptId,
1379 baseline,
1380 providerTurnOrdinal: 1,
1381 message: {
1382 messageId: MessageId.make("message:run-execution-forked-baseline"),
1383 text: "Wait for the baseline.",
1384 attachments: [],
1385 createdBy: "user",
1386 creationSource: "web",
1387 },
1388 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
1389 runtimePolicy: {
1390 runtimeMode: "full-access",
1391 interactionMode: "default",
1392 cwd: process.cwd(),
1393 approvalPolicy: "never",
1394 sandboxPolicy: {
1395 type: "readOnly",
1396 access: { type: "fullAccess" },
1397 networkAccess: false,
1398 },
1399 },
1400 })
1401 .pipe(Effect.forkChild);
1402 yield* TestClock.adjust("1 second");
1403 yield* Fiber.join(started);
1404 }).pipe(Effect.provide(layerTest));
1405
1406 assert.deepEqual(yield* Ref.get(order), ["baseline", "prompt"]);
1407 }),
1408 );
1409
1410 it.effect.each(["failure", "interruption", "stale-attempt", "start-guard"] as const)(
1411 "handles %s before the provider turn starts",
1412 (scenario) =>
1413 Effect.gen(function* () {
1414 const threadId = ThreadId.make("thread:run-execution-settings-failure");
1415 const runId = RunId.make("run:run-execution-settings-failure");
1416 const attemptId = RunAttemptId.make("attempt:run-execution-settings-failure");
1417 const providerInstanceId = ProviderInstanceId.make("codex");
1418 const providerSessionId = ProviderSessionId.make("session:run-execution-settings-failure");
1419 const providerThreadId = ProviderThreadId.make(
1420 "provider-thread:run-execution-settings-failure",
1421 );
1422 const rootNodeId = NodeId.make("node:run-execution-settings-failure");
1423 const checkpointScope = {
1424 id: CheckpointScopeId.make("checkpoint-scope:run-execution-settings-failure"),
1425 } as OrchestrationV2CheckpointScope;
1426 const providerStarts = yield* Ref.make(0);
1427 const refreshes = yield* Ref.make(0);
1428 const guardedWrites = yield* Ref.make(0);
1429 const writes = yield* Ref.make<ReadonlyArray<ReadonlyArray<OrchestrationV2DomainEvent>>>([]);
1430 const layerTest = RunExecutionService.layer.pipe(
1431 Layer.provide(
1432 Layer.mergeAll(
1433 McpAppModelContext.layerEmpty,
1434 PreviewAgentActivity.layer,
1435 Layer.mock(CheckpointService.CheckpointServiceV2)({
1436 captureBaseline: () =>
1437 scenario === "start-guard" ? Effect.void : Effect.die("not reached"),
1438 }),
1439 layerTurnItemPositionsTest,
1440 mockEventSink({
1441 writeIfRunCurrent: (input) =>
1442 Effect.gen(function* () {
1443 assert.equal(input.threadId, threadId);
1444 assert.equal(input.runId, runId);
1445 assert.equal(input.activeAttemptId, attemptId);
1446 assert.equal(input.expectedStatus, "running");
1447 yield* Ref.update(guardedWrites, (count) => count + 1);
1448 if (scenario === "stale-attempt") {
1449 return { committed: false, storedEvents: [] };
1450 }
1451 yield* Ref.update(writes, (current) => [...current, input.events]);
1452 return { committed: true, storedEvents: [] };
1453 }),
1454 }),
1455 IdAllocator.layer,
1456 ThreadCommandExecutor.layer,
1457 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
1458 ingestNormalized: () => Effect.succeed([]),
1459 }),
1460 scenario === "start-guard"
1461 ? ServerSettings.layerTest()
1462 : Layer.mock(ServerSettings.ServerSettingsService)({
1463 getSettings:
1464 scenario === "interruption"
1465 ? Effect.interrupt
1466 : Effect.fail(
1467 new ServerSettingsError({
1468 settingsPath: "<test>",
1469 operation: "read-file",
1470 cause: new Error("settings read failed"),
1471 }),
1472 ),
1473 }),
1474 Layer.succeed(RunFinalizationService.RunFinalizationObserver, {
1475 refresh: () => Effect.void,
1476 refreshAfterTurn: () => Ref.update(refreshes, (count) => count + 1),
1477 }),
1478 ),
1479 ),
1480 );
1481
1482 const result = yield* Effect.gen(function* () {
1483 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
1484 yield* runExecution.startRootRun({
1485 commandId: CommandId.make("command:run-execution-settings-failure"),
1486 appThread: rootAppThread(threadId),
1487 providerSessionId,
1488 session: {
1489 events: Stream.never,
1490 startTurn: () => Ref.update(providerStarts, (count) => count + 1),
1491 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
1492 run: {
1493 id: runId,
1494 threadId,
1495 ordinal: 1,
1496 providerInstanceId,
1497 status: "running",
1498 } as OrchestrationV2Run,
1499 rootNode: { id: rootNodeId, status: "running" } as OrchestrationV2ExecutionNode,
1500 checkpointScope,
1501 providerThread: {
1502 id: providerThreadId,
1503 driver,
1504 } as OrchestrationV2ProviderThread,
1505 attempt: {
1506 id: attemptId,
1507 providerTurnId: null,
1508 status: "running",
1509 } as OrchestrationV2RunAttempt,
1510 attemptId,
1511 providerTurnOrdinal: 1,
1512 ...(scenario === "start-guard"
1513 ? { shouldStartProviderTurn: () => Effect.succeed(false) }
1514 : {}),
1515 message: {
1516 messageId: MessageId.make("message:run-execution-settings-failure"),
1517 text: "Start after settings fail.",
1518 attachments: [],
1519 createdBy: "user",
1520 creationSource: "web",
1521 },
1522 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
1523 runtimePolicy: {
1524 runtimeMode: "full-access",
1525 interactionMode: "default",
1526 cwd: process.cwd(),
1527 approvalPolicy: "never",
1528 sandboxPolicy: {
1529 type: "readOnly",
1530 access: { type: "fullAccess" },
1531 networkAccess: false,
1532 },
1533 },
1534 });
1535 }).pipe(Effect.provide(layerTest), Effect.exit);
1536
1537 assert.equal(yield* Ref.get(providerStarts), 0);
1538 const events = (yield* Ref.get(writes)).flat();
1539 if (scenario === "interruption") {
1540 assert.isTrue(Exit.isFailure(result));
1541 if (Exit.isFailure(result)) assert.isTrue(Cause.hasInterruptsOnly(result.cause));
1542 assert.equal(yield* Ref.get(guardedWrites), 0);
1543 assert.equal(yield* Ref.get(refreshes), 0);
1544 assert.isEmpty(events);
1545 return;
1546 }
1547 assert.isTrue(Exit.isSuccess(result));
1548 if (scenario === "start-guard") {
1549 assert.equal(yield* Ref.get(guardedWrites), 0);
1550 assert.equal(yield* Ref.get(refreshes), 0);
1551 assert.isEmpty(events);
1552 return;
1553 }
1554 assert.equal(yield* Ref.get(guardedWrites), 1);
1555 if (scenario === "stale-attempt") {
1556 assert.equal(yield* Ref.get(refreshes), 0);
1557 assert.isEmpty(events);
1558 return;
1559 }
1560 assert.equal(yield* Ref.get(refreshes), 1);
1561 assert.deepEqual(
1562 events
1563 .filter(
1564 (event) =>
1565 event.type === "run.updated" ||
1566 event.type === "run-attempt.updated" ||
1567 event.type === "node.updated",
1568 )
1569 .map((event) => event.payload.status),
1570 ["failed", "failed", "failed"],
1571 );
1572 const errorItem = events.find(
1573 (event) => event.type === "turn-item.updated" && event.payload.type === "error",
1574 );
1575 assert.isDefined(errorItem);
1576 if (errorItem?.type === "turn-item.updated" && errorItem.payload.type === "error") {
1577 assert.equal(errorItem.payload.failure.message, "Run preparation failed.");
1578 }
1579 }),
1580 );
1581
1582 it.effect("ingests new member threads of an owned child after the root turn terminalizes", () =>
1583 Effect.gen(function* () {
1584 const threadId = ThreadId.make("thread:run-execution-late-child");
1585 const childThreadId = ThreadId.make("thread:run-execution-late-child:child");
1586 const memberThreadId = ThreadId.make("thread:run-execution-late-child:member");
1587 const runId = RunId.make("run:run-execution-late-child");
1588 const attemptId = RunAttemptId.make("attempt:run-execution-late-child");
1589 const providerInstanceId = ProviderInstanceId.make("codex");
1590 const providerSessionId = ProviderSessionId.make("session:run-execution-late-child");
1591 const providerThreadId = ProviderThreadId.make("provider-thread:run-execution-late-child");
1592 const childProviderThreadId = ProviderThreadId.make(
1593 "provider-thread:run-execution-late-child:child",
1594 );
1595 const rootProviderTurnId = ProviderTurnId.make("provider-turn:run-execution-late-child");
1596 const childProviderTurnId = ProviderTurnId.make("provider-turn:run-execution-late-child:child");
1597 const rootNodeId = NodeId.make("node:run-execution-late-child");
1598 const childNodeId = NodeId.make("node:run-execution-late-child:child");
1599 const subagentNodeId = NodeId.make("node:run-execution-late-child:subagent");
1600 const childMessageIngested = yield* Deferred.make<void>();
1601 const order = yield* Ref.make<ReadonlyArray<string>>([]);
1602 const origins = yield* Ref.make<ReadonlyArray<readonly [string, unknown, unknown]>>([]);
1603 const requests = yield* Ref.make<ReadonlyArray<string>>([]);
1604 const layerTest = RunExecutionService.layer.pipe(
1605 Layer.provide(
1606 Layer.mergeAll(
1607 Layer.succeed(ProviderUsageAnalytics.ProviderTurnAnalytics, {
1608 runStarted: (input) =>
1609 Ref.update(requests, (current) => [...current, `started:${input.runId}`]),
1610 turnCompleted: () => Effect.void,
1611 runSettled: (input) =>
1612 Ref.update(requests, (current) => [
1613 ...current,
1614 `settled:${input.runId}:${input.status}:children-running=${input.childrenRunning}`,
1615 ]),
1616 }),
1617 McpAppModelContext.layerEmpty,
1618 PreviewAgentActivity.layer,
1619 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
1620 layerTurnItemPositionsTest,
1621 mockEventSink({
1622 write: () => Effect.succeed([]),
1623 writeWithEffects: (input) =>
1624 Effect.gen(function* () {
1625 if (
1626 input.events.some(
1627 (event) => event.type === "run.updated" && event.runId === runId,
1628 )
1629 ) {
1630 yield* Ref.update(order, (current) => [...current, "root-finalized"]);
1631 }
1632 return [];
1633 }),
1634 writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }),
1635 }),
1636 IdAllocator.layer,
1637 ThreadCommandExecutor.layer,
1638 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
1639 ingestNormalized: (input) =>
1640 Effect.gen(function* () {
1641 const eventType = input.event.type;
1642 yield* Ref.update(origins, (current) => [
1643 ...current,
1644 [
1645 eventType,
1646 input.analyticsContext?.origin,
1647 input.analyticsContext?.segment,
1648 ] as const,
1649 ]);
1650 if (
1651 input.event.type === "app_thread.created" &&
1652 input.event.appThread.id === memberThreadId
1653 ) {
1654 yield* Ref.update(order, (current) => [...current, "member-thread"]);
1655 }
1656 if (
1657 input.event.type === "message.updated" &&
1658 input.event.message.threadId === memberThreadId
1659 ) {
1660 yield* Ref.update(order, (current) => [...current, "child-message"]);
1661 yield* Deferred.succeed(childMessageIngested, undefined).pipe(Effect.ignore);
1662 }
1663 return [];
1664 }),
1665 }),
1666 ServerSettings.layerTest(),
1667 ),
1668 ),
1669 );
1670 const events: ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event> = [
1671 {
1672 type: "app_thread.created",
1673 driver,
1674 appThread: {
1675 id: childThreadId,
1676 lineage: {
1677 parentThreadId: threadId,
1678 relationshipToParent: "subagent",
1679 rootThreadId: threadId,
1680 },
1681 },
1682 } as ProviderAdapter.ProviderAdapterV2Event,
1683 {
1684 type: "provider_thread.updated",
1685 driver,
1686 providerThread: {
1687 id: childProviderThreadId,
1688 appThreadId: childThreadId,
1689 },
1690 } as ProviderAdapter.ProviderAdapterV2Event,
1691 {
1692 type: "subagent.updated",
1693 driver,
1694 subagent: {
1695 ...makeRunOwnedSubagentFixture({
1696 ids: backgroundScenarioIds("run-execution-late-child"),
1697 providerInstanceId,
1698 childThreadId,
1699 driver,
1700 status: "running",
1701 }),
1702 workflow: {
1703 phases: [],
1704 agents: [
1705 { index: 0, label: "Reviewer", state: "running", childThreadId: memberThreadId },
1706 ],
1707 },
1708 },
1709 },
1710 {
1711 type: "provider_turn.updated",
1712 driver,
1713 threadId: childThreadId,
1714 providerTurn: {
1715 id: childProviderTurnId,
1716 providerThreadId: childProviderThreadId,
1717 nodeId: childNodeId,
1718 runAttemptId: null,
1719 status: "running",
1720 },
1721 } as ProviderAdapter.ProviderAdapterV2Event,
1722 {
1723 type: "turn.terminal",
1724 driver,
1725 providerThreadId,
1726 providerTurnId: rootProviderTurnId,
1727 runOrdinal: 1,
1728 status: "completed",
1729 failure: null,
1730 threadDisposition: "reusable",
1731 },
1732 {
1733 type: "app_thread.created",
1734 driver,
1735 appThread: {
1736 id: memberThreadId,
1737 lineage: {
1738 parentThreadId: childThreadId,
1739 relationshipToParent: "subagent",
1740 rootThreadId: threadId,
1741 },
1742 },
1743 } as ProviderAdapter.ProviderAdapterV2Event,
1744 {
1745 type: "message.updated",
1746 driver,
1747 message: {
1748 id: MessageId.make("message:run-execution-late-child:child"),
1749 threadId: memberThreadId,
1750 runId: null,
1751 nodeId: childNodeId,
1752 role: "assistant",
1753 text: "Hello.",
1754 streaming: false,
1755 },
1756 } as ProviderAdapter.ProviderAdapterV2Event,
1757 {
1758 type: "provider_turn.updated",
1759 driver,
1760 threadId: childThreadId,
1761 providerTurn: {
1762 id: childProviderTurnId,
1763 providerThreadId: childProviderThreadId,
1764 nodeId: childNodeId,
1765 runAttemptId: null,
1766 status: "completed",
1767 },
1768 } as ProviderAdapter.ProviderAdapterV2Event,
1769 {
1770 type: "subagent.updated",
1771 driver,
1772 subagent: {
1773 id: subagentNodeId,
1774 threadId,
1775 runId,
1776 status: "completed",
1777 },
1778 } as ProviderAdapter.ProviderAdapterV2Event,
1779 ];
1780
1781 yield* Effect.gen(function* () {
1782 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
1783 yield* runExecution.startRootRun({
1784 commandId: CommandId.make("command:run-execution-late-child"),
1785 appThread: rootAppThread(threadId),
1786 providerSessionId,
1787 session: {
1788 events: Stream.fromIterable(events),
1789 startTurn: () => Effect.void,
1790 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
1791 run: {
1792 id: runId,
1793 threadId,
1794 ordinal: 1,
1795 providerInstanceId,
1796 } as OrchestrationV2Run,
1797 rootNode: { id: rootNodeId } as OrchestrationV2ExecutionNode,
1798 checkpointScope: {
1799 id: CheckpointScopeId.make("checkpoint-scope:run-execution-late-child"),
1800 } as OrchestrationV2CheckpointScope,
1801 providerThread: {
1802 id: providerThreadId,
1803 driver,
1804 } as OrchestrationV2ProviderThread,
1805 attempt: {
1806 id: attemptId,
1807 providerTurnId: rootProviderTurnId,
1808 reason: "initial",
1809 } as OrchestrationV2RunAttempt,
1810 attemptId,
1811 providerTurnOrdinal: 1,
1812 message: {
1813 messageId: MessageId.make("message:run-execution-late-child:user"),
1814 text: "Spawn a child and finish.",
1815 attachments: [],
1816 createdBy: "user",
1817 creationSource: "web",
1818 },
1819 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
1820 runtimePolicy: {
1821 runtimeMode: "full-access",
1822 interactionMode: "default",
1823 cwd: process.cwd(),
1824 approvalPolicy: "never",
1825 sandboxPolicy: {
1826 type: "readOnly",
1827 access: { type: "fullAccess" },
1828 networkAccess: false,
1829 },
1830 },
1831 });
1832 }).pipe(Effect.provide(layerTest));
1833
1834 const observed = yield* Deferred.await(childMessageIngested).pipe(
1835 Effect.timeoutOption("2 seconds"),
1836 );
1837 assert.isTrue(Option.isSome(observed), "child message was not ingested after root terminal");
1838 assert.deepEqual(yield* Ref.get(order), ["root-finalized", "member-thread", "child-message"]);
1839 // The run's message started the root; the parent agent started the child's turn.
1840 const recorded = yield* Ref.get(origins);
1841 assert.deepEqual(recorded[0]?.slice(0, 2), [
1842 "app_thread.created",
1843 { createdBy: "user", creationSource: "web", scheduled: false, subagent: false },
1844 ]);
1845 assert.deepEqual(
1846 recorded.find(([type]) => type === "provider_turn.updated"),
1847 [
1848 "provider_turn.updated",
1849 { createdBy: "agent", creationSource: "provider", scheduled: false, subagent: true },
1850 "native_child",
1851 ],
1852 );
1853 // The child outlives the root, so the request summary says its later turns are missing.
1854 assert.deepEqual(yield* Ref.get(requests), [
1855 `started:${runId}`,
1856 `settled:${runId}:completed:children-running=true`,
1857 ]);
1858 }),
1859 );
1860
1861 it.effect("ends a nested native subagent and its thread when the root run is interrupted", () =>
1862 Effect.gen(function* () {
1863 const key = "run-execution-nested-native";
1864 const threadId = ThreadId.make(`thread:${key}`);
1865 const childThreadId = ThreadId.make(`thread:${key}:child`);
1866 const grandchildThreadId = ThreadId.make(`thread:${key}:grandchild`);
1867 const runId = RunId.make(`run:${key}`);
1868 const attemptId = RunAttemptId.make(`attempt:${key}`);
1869 const providerInstanceId = ProviderInstanceId.make("codex");
1870 const providerThreadId = ProviderThreadId.make(`provider-thread:${key}`);
1871 const rootProviderTurnId = ProviderTurnId.make(`provider-turn:${key}`);
1872 const nestedId = NodeId.make(`node:${key}:nested`);
1873 const grandchildRootId = NodeId.make(`node:${key}:grandchild-root`);
1874 const finalized = yield* Deferred.make<ReadonlyArray<OrchestrationV2DomainEvent>>();
1875 const layerTest = RunExecutionService.layer.pipe(
1876 Layer.provide(
1877 Layer.mergeAll(
1878 McpAppModelContext.layerEmpty,
1879 PreviewAgentActivity.layer,
1880 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
1881 layerTurnItemPositionsTest,
1882 mockEventSink({
1883 write: () => Effect.succeed([]),
1884 writeWithEffects: (input) =>
1885 Effect.as(
1886 input.events.some((event) => event.type === "run.updated")
1887 ? Deferred.succeed(finalized, input.events)
1888 : Effect.void,
1889 [],
1890 ),
1891 }),
1892 IdAllocator.layer,
1893 ThreadCommandExecutor.layer,
1894 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
1895 ingestNormalized: () => Effect.succeed([]),
1896 }),
1897 ServerSettings.layerTest(),
1898 ),
1899 ),
1900 );
1901 const subagentThread = (id: ThreadId, parentThreadId: ThreadId) =>
1902 ({
1903 type: "app_thread.created",
1904 driver,
1905 appThread: {
1906 id,
1907 lineage: { parentThreadId, relationshipToParent: "subagent", rootThreadId: threadId },
1908 },
1909 }) as ProviderAdapter.ProviderAdapterV2Event;
1910 const events: ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event> = [
1911 subagentThread(childThreadId, threadId),
1912 subagentThread(grandchildThreadId, childThreadId),
1913 {
1914 type: "subagent.updated",
1915 driver,
1916 subagent: {
1917 id: nestedId,
1918 threadId: childThreadId,
1919 runId: null,
1920 origin: "provider_native",
1921 childThreadId: grandchildThreadId,
1922 status: "running",
1923 },
1924 } as ProviderAdapter.ProviderAdapterV2Event,
1925 {
1926 type: "node.updated",
1927 driver,
1928 node: {
1929 id: grandchildRootId,
1930 threadId: grandchildThreadId,
1931 runId: null,
1932 kind: "root_turn",
1933 status: "running",
1934 },
1935 } as ProviderAdapter.ProviderAdapterV2Event,
1936 {
1937 type: "turn.terminal",
1938 driver,
1939 providerThreadId,
1940 providerTurnId: rootProviderTurnId,
1941 runOrdinal: 1,
1942 status: "interrupted",
1943 failure: null,
1944 threadDisposition: "reusable",
1945 },
1946 ];
1947
1948 yield* Effect.gen(function* () {
1949 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
1950 yield* runExecution.startRootRun({
1951 commandId: CommandId.make(`command:${key}`),
1952 appThread: rootAppThread(threadId),
1953 providerSessionId: ProviderSessionId.make(`session:${key}`),
1954 session: {
1955 events: Stream.fromIterable(events),
1956 startTurn: () => Effect.void,
1957 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
1958 run: { id: runId, threadId, ordinal: 1, providerInstanceId } as OrchestrationV2Run,
1959 rootNode: { id: NodeId.make(`node:${key}`) } as OrchestrationV2ExecutionNode,
1960 checkpointScope: {
1961 id: CheckpointScopeId.make(`checkpoint-scope:${key}`),
1962 } as OrchestrationV2CheckpointScope,
1963 providerThread: { id: providerThreadId, driver } as OrchestrationV2ProviderThread,
1964 attempt: { id: attemptId, providerTurnId: rootProviderTurnId } as OrchestrationV2RunAttempt,
1965 attemptId,
1966 providerTurnOrdinal: 1,
1967 message: {
1968 messageId: MessageId.make(`message:${key}:user`),
1969 text: "Spawn a nested subagent.",
1970 attachments: [],
1971 createdBy: "user",
1972 creationSource: "web",
1973 },
1974 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
1975 runtimePolicy: {
1976 runtimeMode: "full-access",
1977 interactionMode: "default",
1978 cwd: process.cwd(),
1979 approvalPolicy: "never",
1980 sandboxPolicy: { type: "readOnly", access: { type: "fullAccess" }, networkAccess: false },
1981 },
1982 });
1983 }).pipe(Effect.provide(layerTest));
1984
1985 const ended = (yield* Deferred.await(finalized)).flatMap((event) =>
1986 event.type === "subagent.updated" || event.type === "node.updated"
1987 ? [[event.type, event.payload.id, event.payload.status]]
1988 : [],
1989 );
1990 assert.deepInclude(ended, ["subagent.updated", nestedId, "interrupted"]);
1991 assert.deepInclude(ended, ["node.updated", grandchildRootId, "interrupted"]);
1992 }),
1993 );
1994
1995 it.effect("drains queued child output after the root loses provider-thread ownership", () =>
1996 Effect.gen(function* () {
1997 const childEvents: Array<string> = [];
1998 yield* runBackgroundItemScenario(
1999 "queued-child-output",
2000 (ids) => {
2001 const childProviderThreadId = ProviderThreadId.make(
2002 "provider-thread:queued-child-output:child",
2003 );
2004 const snapshot = (child: boolean, pendingBackgroundWork: boolean) => ({
2005 type: "provider_thread.updated" as const,
2006 driver,
2007 providerThread: {
2008 id: child ? childProviderThreadId : ids.providerThreadId,
2009 driver,
2010 providerInstanceId: ProviderInstanceId.make("codex"),
2011 providerSessionId: null,
2012 appThreadId: child ? ids.childThreadId : ids.threadId,
2013 ownerNodeId: null,
2014 nativeThreadRef: null,
2015 nativeConversationHeadRef: null,
2016 status: "idle",
2017 firstRunOrdinal: 1,
2018 lastRunOrdinal: 1,
2019 handoffIds: [],
2020 forkedFrom: null,
2021 pendingBackgroundTasks: [],
2022 createdAt: DateTime.makeUnsafe(0),
2023 updatedAt: DateTime.makeUnsafe(0),
2024 } satisfies OrchestrationV2ProviderThread,
2025 pendingBackgroundWork,
2026 });
2027 const childTurn = (status: "running" | "completed") => ({
2028 type: "provider_turn.updated" as const,
2029 driver,
2030 threadId: ids.childThreadId,
2031 providerTurn: {
2032 id: ProviderTurnId.make("provider-turn:queued-child-output:child"),
2033 providerThreadId: childProviderThreadId,
2034 runAttemptId: null,
2035 ordinal: 2,
2036 status,
2037 } as OrchestrationV2ProviderTurn,
2038 });
2039 const message = (text: string) =>
2040 ({
2041 type: "message.updated",
2042 driver,
2043 message: { threadId: ids.childThreadId, runId: null, text },
2044 }) as ProviderAdapter.ProviderAdapterV2Event;
2045 return [
2046 childThreadCreatedEvent(ids),
2047 snapshot(false, true),
2048 snapshot(true, true),
2049 subagentEvent(ids, "running"),
2050 rootTerminalEvent(ids, "completed"),
2051 snapshot(false, true),
2052 subagentEvent(ids, "completed"),
2053 childTurn("running"),
2054 message("GRANDCHILD_OK"),
2055 childTurn("completed"),
2056 snapshot(true, false),
2057 snapshot(false, false),
2058 message("after-drain"),
2059 ];
2060 },
2061 {
2062 keepEventStreamOpen: true,
2063 hasPendingBackgroundWorkForThread: () => Effect.succeed(false),
2064 providerThreadOwnerLost: true,
2065 onIngest: (event) =>
2066 Effect.sync(() => {
2067 if (event.type === "provider_turn.updated") {
2068 childEvents.push(event.providerTurn.status);
2069 }
2070 if (event.type === "message.updated") childEvents.push(event.message.text);
2071 }),
2072 },
2073 );
2074 assert.deepEqual(childEvents, ["running", "GRANDCHILD_OK", "completed"]);
2075 }),
2076 );
2077
2078 it.effect("keeps ingesting until queued background work clears after the probe goes idle", () =>
2079 Effect.gen(function* () {
2080 // The adapter's probe runs ahead of its event queue, so only the queued update may release.
2081 // A child transferred to another run takes its pending work with it.
2082 const pending = yield* Ref.make(true);
2083 const consumed = yield* Deferred.make<void>();
2084 const result = yield* Effect.raceFirst(
2085 runBackgroundItemScenario(
2086 "queued-background-work",
2087 (ids) => {
2088 const childProviderThreadId = ProviderThreadId.make(`${ids.providerThreadId}:child`);
2089 const childProviderTurnId = ProviderTurnId.make(`${ids.rootProviderTurnId}:child`);
2090 const running = subagentEvent(ids, "running");
2091 if (running.type !== "subagent.updated") throw new Error("Missing subagent");
2092 const linked = {
2093 ...running,
2094 subagent: {
2095 ...running.subagent,
2096 origin: "provider_native" as const,
2097 childThreadId: ids.childThreadId,
2098 providerThreadId: childProviderThreadId,
2099 },
2100 };
2101 const backgroundWork = (providerTurnId: ProviderTurnId, pending: boolean) =>
2102 ({
2103 type: "background_work.updated",
2104 driver,
2105 providerTurnId,
2106 pending,
2107 }) satisfies ProviderAdapter.ProviderAdapterV2Event;
2108 return [
2109 childThreadCreatedEvent(ids),
2110 linked,
2111 {
2112 type: "provider_turn.updated",
2113 driver,
2114 threadId: ids.childThreadId,
2115 providerTurn: {
2116 id: childProviderTurnId,
2117 providerThreadId: childProviderThreadId,
2118 runAttemptId: null,
2119 status: "completed",
2120 },
2121 } as ProviderAdapter.ProviderAdapterV2Event,
2122 backgroundWork(childProviderTurnId, true),
2123 backgroundWork(ids.rootProviderTurnId, true),
2124 rootTerminalEvent(ids, "completed"),
2125 { ...linked, subagent: { ...linked.subagent, runId: RunId.make("run:replacement") } },
2126 backgroundTurnItemEvent(ids, "dynamic_tool", "completed", 1),
2127 backgroundWork(ids.rootProviderTurnId, false),
2128 ];
2129 },
2130 {
2131 keepEventStreamOpen: true,
2132 hasPendingBackgroundWorkForThread: () => Ref.get(pending),
2133 onFinalized: () => Ref.set(pending, false),
2134 onEventsConsumed: Deferred.succeed(consumed, undefined).pipe(Effect.asVoid),
2135 },
2136 ).pipe(Effect.asSome),
2137 Deferred.await(consumed).pipe(Effect.as(Option.none())),
2138 );
2139 assert.deepEqual(
2140 result,
2141 Option.some(["subagent:running", "root-finalized", "turn_item:completed"]),
2142 );
2143 }),
2144 );
2145
2146 it.effect("keeps ingesting while the last stored root snapshot reports pending work", () =>
2147 Effect.gen(function* () {
2148 // The live probe is already idle, as when it reads adapter state ahead of the queued events.
2149 const consumed = yield* Deferred.make<void>();
2150 const result = yield* Effect.raceFirst(
2151 runBackgroundItemScenario(
2152 "root-snapshot-pending-work",
2153 (ids) => {
2154 const snapshot = (pendingBackgroundWork: boolean) =>
2155 ({
2156 type: "provider_thread.updated",
2157 driver,
2158 providerThread: { id: ids.providerThreadId, appThreadId: ids.threadId },
2159 pendingBackgroundWork,
2160 }) as ProviderAdapter.ProviderAdapterV2Event;
2161 return [
2162 snapshot(true),
2163 rootTerminalEvent(ids, "completed"),
2164 backgroundTurnItemEvent(ids, "dynamic_tool", "completed", 1),
2165 snapshot(false),
2166 ];
2167 },
2168 {
2169 keepEventStreamOpen: true,
2170 hasPendingBackgroundWorkForThread: () => Effect.succeed(false),
2171 onEventsConsumed: Deferred.succeed(consumed, undefined).pipe(Effect.asVoid),
2172 },
2173 ).pipe(Effect.asSome),
2174 Deferred.await(consumed).pipe(Effect.as(Option.none())),
2175 );
2176 assert.deepEqual(result, Option.some(["root-finalized", "turn_item:completed"]));
2177 }),
2178 );
2179
2180 it.effect("keeps ingesting a child thread's late background item completion", () =>
2181 Effect.gen(function* () {
2182 const observed = yield* runBackgroundItemScenario("bg-child-item", (ids) => [
2183 childThreadCreatedEvent(ids),
2184 subagentEvent(ids, "running"),
2185 childBackgroundTurnItemEvent(ids, "running", 1),
2186 rootTerminalEvent(ids, "completed"),
2187 subagentEvent(ids, "completed"),
2188 childBackgroundTurnItemEvent(ids, "completed", 2),
2189 ]);
2190 assert.deepEqual(observed, [
2191 "subagent:running",
2192 "turn_item:running",
2193 "root-finalized",
2194 "subagent:completed",
2195 "turn_item:completed",
2196 ]);
2197 }),
2198 );
2199
2200 it.effect.each([null, "run:nested-sdk"] as const)(
2201 "releases a settled run when its live child transfers to another run (%s)",
2202 (nestedRunId) =>
2203 Effect.gen(function* () {
2204 const consumed = yield* Deferred.make<void>();
2205 const result = yield* Effect.raceFirst(
2206 runBackgroundItemScenario(
2207 "child-transfer-release",
2208 (ids) => {
2209 const running = subagentEvent(ids, "running");
2210 if (running.type !== "subagent.updated") throw new Error("Missing subagent");
2211 const linked = {
2212 ...running,
2213 subagent: {
2214 ...running.subagent,
2215 origin: "provider_native" as const,
2216 childThreadId: ids.childThreadId,
2217 providerThreadId: null,
2218 },
2219 };
2220 return [
2221 childThreadCreatedEvent(ids),
2222 linked,
2223 {
2224 ...linked,
2225 subagent: {
2226 ...linked.subagent,
2227 id: NodeId.make("node:child-transfer-release:nested"),
2228 threadId: ids.childThreadId,
2229 runId: nestedRunId === null ? null : RunId.make(nestedRunId),
2230 childThreadId: null,
2231 },
2232 },
2233 childBackgroundTurnItemEvent(ids, "running", 1),
2234 rootTerminalEvent(ids, "completed"),
2235 { ...linked, subagent: { ...linked.subagent, runId: RunId.make("run:replacement") } },
2236 ];
2237 },
2238 {
2239 keepEventStreamOpen: true,
2240 onEventsConsumed: Deferred.succeed(consumed, undefined).pipe(Effect.asVoid),
2241 },
2242 ).pipe(Effect.asSome),
2243 Deferred.await(consumed).pipe(Effect.as(Option.none())),
2244 );
2245 assert.isTrue(
2246 Option.isSome(result),
2247 "the transferred child must release its prior subscription",
2248 );
2249 if (Option.isSome(result))
2250 assert.deepEqual(result.value, [
2251 "subagent:running",
2252 "subagent:running",
2253 "turn_item:running",
2254 "root-finalized",
2255 ]);
2256 }),
2257 );
2258
2259 it.effect.each(["none", "sibling", "grandchild"] as const)(
2260 "drops a transferred child's pending snapshot but keeps other owned pending work (%s)",
2261 (otherPending) =>
2262 Effect.gen(function* () {
2263 const consumed = yield* Deferred.make<void>();
2264 const result = yield* Effect.raceFirst(
2265 runBackgroundItemScenario(
2266 "child-transfer-pending",
2267 (ids) => {
2268 const childProviderThreadId = ProviderThreadId.make(
2269 "provider-thread:child-transfer-pending:child",
2270 );
2271 const otherThreadId = ThreadId.make("thread:child-transfer-pending:other");
2272 const snapshot = (
2273 providerThreadId: ProviderThreadId,
2274 appThreadId: ThreadId,
2275 pendingBackgroundWork: boolean,
2276 ) => ({
2277 type: "provider_thread.updated" as const,
2278 driver,
2279 providerThread: {
2280 id: providerThreadId,
2281 appThreadId,
2282 status: "idle",
2283 } as OrchestrationV2ProviderThread,
2284 pendingBackgroundWork,
2285 });
2286 const running = subagentEvent(ids, "running");
2287 if (running.type !== "subagent.updated") throw new Error("Missing subagent");
2288 const linked = {
2289 ...running,
2290 subagent: {
2291 ...running.subagent,
2292 origin: "provider_native" as const,
2293 childThreadId: ids.childThreadId,
2294 providerThreadId: childProviderThreadId,
2295 },
2296 };
2297 // A grandchild stays with this run when its parent moves; a sibling was never the child's.
2298 const other =
2299 otherPending === "none"
2300 ? []
2301 : [
2302 {
2303 ...childThreadCreatedEvent(ids),
2304 appThread: {
2305 id: otherThreadId,
2306 lineage: {
2307 parentThreadId:
2308 otherPending === "grandchild" ? ids.childThreadId : ids.threadId,
2309 relationshipToParent: "subagent",
2310 rootThreadId: ids.threadId,
2311 },
2312 },
2313 } as ProviderAdapter.ProviderAdapterV2Event,
2314 snapshot(
2315 ProviderThreadId.make("provider-thread:child-transfer-pending:other"),
2316 otherThreadId,
2317 true,
2318 ),
2319 ];
2320 return [
2321 childThreadCreatedEvent(ids),
2322 linked,
2323 snapshot(childProviderThreadId, ids.childThreadId, true),
2324 ...other,
2325 rootTerminalEvent(ids, "completed"),
2326 { ...linked, subagent: { ...linked.subagent, runId: RunId.make("run:replacement") } },
2327 snapshot(childProviderThreadId, ids.childThreadId, false),
2328 ];
2329 },
2330 {
2331 keepEventStreamOpen: true,
2332 hasPendingBackgroundWorkForThread: () => Effect.succeed(false),
2333 onEventsConsumed: Deferred.succeed(consumed, undefined).pipe(Effect.asVoid),
2334 },
2335 ).pipe(Effect.asSome),
2336 Deferred.await(consumed).pipe(Effect.as(Option.none())),
2337 );
2338 assert.strictEqual(
2339 Option.isSome(result),
2340 otherPending === "none",
2341 "only pending work this run still owns may keep the prior subscription open",
2342 );
2343 }),
2344 );
2345
2346 // The handover event is not this run's, yet it is what settles the run's agent turns.
2347 it.effect(
2348 "ends preview activity when a transferred child settles a run kept open by background work",
2349 () =>
2350 Effect.gen(function* () {
2351 const active = yield* Deferred.make<boolean>();
2352 yield* runBackgroundItemScenario(
2353 "child-transfer-preview",
2354 (ids) => {
2355 const running = subagentEvent(ids, "running");
2356 if (running.type !== "subagent.updated") throw new Error("Missing subagent");
2357 const linked = {
2358 ...running,
2359 subagent: {
2360 ...running.subagent,
2361 origin: "provider_native" as const,
2362 childThreadId: ids.childThreadId,
2363 providerThreadId: null,
2364 },
2365 };
2366 return [
2367 childThreadCreatedEvent(ids),
2368 linked,
2369 backgroundTurnItemEvent(ids, "command_execution", "running", 1),
2370 rootTerminalEvent(ids, "completed"),
2371 { ...linked, subagent: { ...linked.subagent, runId: RunId.make("run:replacement") } },
2372 ];
2373 },
2374 {
2375 keepEventStreamOpen: true,
2376 afterEvents: (activity) =>
2377 activity
2378 .active(backgroundScenarioIds("child-transfer-preview").threadId)
2379 .pipe(Effect.flatMap((value) => Deferred.succeed(active, value))),
2380 },
2381 ).pipe(Effect.forkScoped);
2382 assert.isFalse(yield* Deferred.await(active));
2383 }),
2384 );
2385
2386 it.effect("keeps ingesting a failed root's background item when its work continues", () =>
2387 Effect.gen(function* () {
2388 const observed = yield* runBackgroundItemScenario("bg-failed-continues", (ids) => [
2389 backgroundTurnItemEvent(ids, "command_execution", "running", 1),
2390 { ...rootTerminalEvent(ids, "failed"), childTurnsContinue: true },
2391 backgroundTurnItemEvent(ids, "command_execution", "completed", 2),
2392 ]);
2393 assert.deepEqual(observed, ["turn_item:running", "root-finalized", "turn_item:completed"]);
2394 }),
2395 );
2396
2397 it.effect.each(["interrupted", "failed"] as const)(
2398 "does not pin ingestion on background items when the root turn is %s",
2399 (status) =>
2400 Effect.gen(function* () {
2401 const observed = yield* runBackgroundItemScenario(`bg-${status}`, (ids) => [
2402 backgroundTurnItemEvent(ids, "command_execution", "running", 1),
2403 rootTerminalEvent(ids, status),
2404 backgroundTurnItemEvent(ids, "command_execution", "completed", 2),
2405 ]);
2406 assert.deepEqual(observed, ["turn_item:running", "root-finalized"]);
2407 }),
2408 );
2409
2410 it.effect("seeds inherited background items before their next update", () =>
2411 Effect.gen(function* () {
2412 const key = "inherited-background-seeded";
2413 const priorRunId = RunId.make(`run:${key}:prior`);
2414 const observed = yield* runBackgroundItemScenario(
2415 key,
2416 (ids) => [
2417 rootTerminalEvent(ids, "completed"),
2418 backgroundTurnItemEventForRun(ids, priorRunId, "subagent", "completed", 1),
2419 ],
2420 {
2421 loadInheritedBackgroundTurnItems: () =>
2422 Effect.succeed([{ id: TurnItemId.make(`turn-item:${key}`), runId: priorRunId }]),
2423 },
2424 );
2425
2426 assert.deepEqual(observed, ["root-finalized", "turn_item:completed"]);
2427 }),
2428 );
2429
2430 it.effect("refreshes inherited background items after event subscription", () =>
2431 Effect.gen(function* () {
2432 const key = "inherited-background-subscription-refresh";
2433 const ids = backgroundScenarioIds(key);
2434 const priorRunId = RunId.make(`run:${key}:prior`);
2435 const itemStatus = yield* Ref.make<OrchestrationV2TurnItem["status"]>("running");
2436 const loadInheritedBackgroundTurnItems = () =>
2437 Ref.get(itemStatus).pipe(
2438 Effect.map((status) =>
2439 RunExecutionService.selectInheritedBackgroundTurnItems({
2440 threadId: ids.threadId,
2441 currentProviderThreadId: ids.providerThreadId,
2442 currentRunOrdinal: 2,
2443 runs: [
2444 {
2445 id: priorRunId,
2446 threadId: ids.threadId,
2447 ordinal: 1,
2448 status: "interrupted",
2449 } as OrchestrationV2Run,
2450 ],
2451 turnItems: [
2452 {
2453 id: ids.itemId,
2454 threadId: ids.threadId,
2455 runId: priorRunId,
2456 providerThreadId: ids.providerThreadId,
2457 type: "subagent",
2458 status,
2459 } as OrchestrationV2TurnItem,
2460 ],
2461 }),
2462 ),
2463 );
2464
2465 const observed = yield* runBackgroundItemScenario(
2466 key,
2467 (scenarioIds) => [rootTerminalEvent(scenarioIds, "completed")],
2468 {
2469 keepEventStreamOpen: true,
2470 loadInheritedBackgroundTurnItems,
2471 onSubscribe: Ref.set(itemStatus, "completed"),
2472 },
2473 );
2474
2475 assert.deepEqual(yield* loadInheritedBackgroundTurnItems(), []);
2476 assert.deepEqual(observed, ["root-finalized"]);
2477 }),
2478 );
2479
2480 it.effect(
2481 "cascade-terminalizes run-owned subagent rows on interrupt before root finalization",
2482 () =>
2483 Effect.gen(function* () {
2484 const ids = backgroundScenarioIds("subagent-interrupt-cascade");
2485 const childThreadId = ids.childThreadId;
2486 const unrelatedChildThreadId = ThreadId.make(
2487 "thread:subagent-interrupt-cascade:unrelated-child",
2488 );
2489 const providerInstanceId = ProviderInstanceId.make("codex");
2490 const written = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);
2491 const ingested = yield* Ref.make<ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>>([]);
2492 const ingestionDone = yield* Deferred.make<void>();
2493 const layerTest = RunExecutionService.layer.pipe(
2494 Layer.provide(
2495 Layer.mergeAll(
2496 McpAppModelContext.layerEmpty,
2497 PreviewAgentActivity.layer,
2498 Layer.mock(CheckpointService.CheckpointServiceV2)({
2499 captureBaseline: () => Effect.void,
2500 }),
2501 layerTurnItemPositionsTest,
2502 mockEventSink({
2503 write: () => Effect.succeed([]),
2504 writeWithEffects: (input) =>
2505 Effect.gen(function* () {
2506 yield* Ref.update(written, (current) => [...current, ...input.events]);
2507 return [];
2508 }),
2509 writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }),
2510 }),
2511 IdAllocator.layer,
2512 ThreadCommandExecutor.layer,
2513 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
2514 ingestNormalized: (input) =>
2515 Ref.update(ingested, (current) => [...current, input.event]).pipe(Effect.as([])),
2516 }),
2517 ServerSettings.layerTest(),
2518 ),
2519 ),
2520 );
2521
2522 const runningSubagent = makeRunOwnedSubagentFixture({
2523 ids,
2524 providerInstanceId,
2525 childThreadId,
2526 driver,
2527 status: "running",
2528 });
2529 const runningTurnItem = makeRunOwnedSubagentTurnItemFixture({
2530 ids,
2531 providerInstanceId,
2532 childThreadId,
2533 driver,
2534 status: "running",
2535 });
2536 const runningNode = makeRunOwnedSubagentNodeFixture({
2537 ids,
2538 status: "running",
2539 });
2540 const runningChildNode = makeRunOwnedSubagentChildNodeFixture({
2541 ids,
2542 status: "running",
2543 });
2544 const memberThreadId = ThreadId.make("thread:subagent-interrupt-cascade:member");
2545 const runningMember = {
2546 ...runningSubagent,
2547 id: NodeId.make("node:subagent-interrupt-cascade:member"),
2548 threadId: childThreadId,
2549 runId: null,
2550 parentNodeId: runningChildNode.id,
2551 childThreadId: memberThreadId,
2552 };
2553 const foreignSubagent = {
2554 ...runningMember,
2555 id: NodeId.make("node:subagent-interrupt-cascade:foreign-subagent"),
2556 runId: RunId.make("run:subagent-interrupt-cascade:foreign"),
2557 childThreadId: unrelatedChildThreadId,
2558 };
2559 const runningMemberRoot = {
2560 ...runningChildNode,
2561 id: NodeId.make("node:subagent-interrupt-cascade:member-root"),
2562 threadId: memberThreadId,
2563 rootNodeId: NodeId.make("node:subagent-interrupt-cascade:member-root"),
2564 };
2565 const foreignChildNode = {
2566 ...runningChildNode,
2567 id: NodeId.make("node:subagent-interrupt-cascade:foreign-child"),
2568 runId: foreignSubagent.runId,
2569 };
2570 const runningChildTurnItem = makeLinkedChildTurnItemFixture({
2571 ids,
2572 driver,
2573 type: "command_execution",
2574 });
2575 const foreignChildTurnItem = {
2576 ...runningChildTurnItem,
2577 id: TurnItemId.make("turn-item:subagent-interrupt-cascade:foreign-child"),
2578 runId: foreignSubagent.runId,
2579 };
2580 const suppressedChildAssistantTurnItem = makeLinkedChildTurnItemFixture({
2581 ids: {
2582 ...ids,
2583 childItemId: TurnItemId.make("turn-item:subagent-interrupt-cascade:suppressed-assistant"),
2584 },
2585 driver,
2586 type: "assistant_message",
2587 });
2588 const unrelatedChildNode = {
2589 ...runningChildNode,
2590 id: NodeId.make("node:subagent-interrupt-cascade:unrelated-child"),
2591 threadId: unrelatedChildThreadId,
2592 runId: ids.runId,
2593 rootNodeId: NodeId.make("node:subagent-interrupt-cascade:unrelated-child"),
2594 };
2595 const unrelatedChildTurnItem = {
2596 ...runningChildTurnItem,
2597 id: TurnItemId.make("turn-item:subagent-interrupt-cascade:unrelated-child"),
2598 threadId: unrelatedChildThreadId,
2599 nodeId: unrelatedChildNode.id,
2600 };
2601
2602 yield* Effect.gen(function* () {
2603 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
2604 yield* runExecution.startRootRun({
2605 commandId: CommandId.make("command:subagent-interrupt-cascade"),
2606 appThread: rootAppThread(ids.threadId),
2607 providerSessionId: ProviderSessionId.make("session:subagent-interrupt-cascade"),
2608 session: {
2609 events: Stream.empty,
2610 subscribeEvents: Effect.succeed({
2611 events: Stream.fromIterable([
2612 childThreadCreatedEvent(ids),
2613 {
2614 type: "subagent.updated",
2615 driver,
2616 subagent: runningSubagent,
2617 },
2618 {
2619 type: "node.updated",
2620 driver,
2621 node: runningNode,
2622 },
2623 {
2624 type: "node.updated",
2625 driver,
2626 node: runningChildNode,
2627 },
2628 {
2629 type: "node.updated",
2630 driver,
2631 node: unrelatedChildNode,
2632 },
2633 {
2634 type: "turn_item.updated",
2635 driver,
2636 turnItem: runningTurnItem,
2637 },
2638 {
2639 type: "turn_item.updated",
2640 driver,
2641 turnItem: runningChildTurnItem,
2642 },
2643 {
2644 type: "turn_item.updated",
2645 driver,
2646 turnItem: suppressedChildAssistantTurnItem,
2647 },
2648 {
2649 type: "turn_item.updated",
2650 driver,
2651 turnItem: unrelatedChildTurnItem,
2652 },
2653 { type: "subagent.updated", driver, subagent: runningMember },
2654 { type: "subagent.updated", driver, subagent: foreignSubagent },
2655 { type: "node.updated", driver, node: runningMemberRoot },
2656 { type: "node.updated", driver, node: foreignChildNode },
2657 { type: "turn_item.updated", driver, turnItem: foreignChildTurnItem },
2658 rootTerminalEvent(ids, "interrupted"),
2659 {
2660 type: "subagent.updated",
2661 driver,
2662 subagent: {
2663 ...runningSubagent,
2664 status: "completed" as const,
2665 result: "should-not-apply",
2666 completedAt: runningSubagent.updatedAt,
2667 },
2668 },
2669 {
2670 type: "turn_item.updated",
2671 driver,
2672 turnItem: {
2673 ...runningTurnItem,
2674 status: "completed" as const,
2675 result: "should-not-apply",
2676 completedAt: runningTurnItem.updatedAt,
2677 updatedAt: runningTurnItem.updatedAt,
2678 },
2679 },
2680 {
2681 type: "turn_item.updated",
2682 driver,
2683 turnItem: {
2684 ...runningChildTurnItem,
2685 status: "completed" as const,
2686 completedAt: runningChildTurnItem.updatedAt,
2687 updatedAt: runningChildTurnItem.updatedAt,
2688 },
2689 },
2690 {
2691 type: "node.updated",
2692 driver,
2693 node: {
2694 ...runningChildNode,
2695 status: "completed" as const,
2696 completedAt: runningChildNode.startedAt,
2697 },
2698 },
2699 ] satisfies ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>),
2700 close: Deferred.succeed(ingestionDone, undefined),
2701 }),
2702 startTurn: () => Effect.void,
2703 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
2704 run: {
2705 id: ids.runId,
2706 threadId: ids.threadId,
2707 ordinal: 1,
2708 providerInstanceId,
2709 } as OrchestrationV2Run,
2710 rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode,
2711 checkpointScope: {
2712 id: CheckpointScopeId.make("checkpoint-scope:subagent-interrupt-cascade"),
2713 } as OrchestrationV2CheckpointScope,
2714 providerThread: {
2715 id: ids.providerThreadId,
2716 driver,
2717 } as OrchestrationV2ProviderThread,
2718 attempt: {
2719 id: ids.attemptId,
2720 providerTurnId: ids.rootProviderTurnId,
2721 } as OrchestrationV2RunAttempt,
2722 attemptId: ids.attemptId,
2723 relatedThreadIds: [unrelatedChildThreadId, memberThreadId],
2724 providerTurnOrdinal: 1,
2725 message: {
2726 messageId: MessageId.make("message:subagent-interrupt-cascade:user"),
2727 text: "Spawn a subagent then stop.",
2728 attachments: [],
2729 createdBy: "user",
2730 creationSource: "web",
2731 },
2732 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
2733 runtimePolicy: {
2734 runtimeMode: "full-access",
2735 interactionMode: "default",
2736 cwd: process.cwd(),
2737 approvalPolicy: "never",
2738 sandboxPolicy: {
2739 type: "readOnly",
2740 access: { type: "fullAccess" },
2741 networkAccess: false,
2742 },
2743 },
2744 });
2745 }).pipe(Effect.provide(layerTest));
2746
2747 const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds"));
2748 assert.isTrue(Option.isSome(closed), "event ingestion fiber did not finish");
2749
2750 const events = yield* Ref.get(written);
2751 const subagentEvents = events.flatMap((event) =>
2752 event.type === "subagent.updated" ? [event] : [],
2753 );
2754 const turnItemEvents = events.flatMap((event) => {
2755 if (event.type !== "turn-item.updated" || event.payload.type !== "subagent") {
2756 return [];
2757 }
2758 return [
2759 {
2760 ...event,
2761 payload: event.payload,
2762 },
2763 ];
2764 });
2765 const nodeEvents = events.flatMap((event) =>
2766 event.type === "node.updated" &&
2767 event.payload.status === "interrupted" &&
2768 (event.payload.id === runningNode.id || event.payload.id === runningChildNode.id)
2769 ? [event]
2770 : [],
2771 );
2772 const runUpdatedIndex = events.findIndex((event) => event.type === "run.updated");
2773 assert.isAtLeast(runUpdatedIndex, 0, "root run.updated must be written");
2774
2775 assert.lengthOf(subagentEvents, 2);
2776 const terminalSubagent = subagentEvents.find(
2777 (event) => event.payload.id === runningSubagent.id,
2778 );
2779 const terminalMember = subagentEvents.find((event) => event.payload.id === runningMember.id);
2780 assert.isDefined(terminalMember);
2781 assert.equal(terminalMember.payload.status, "interrupted");
2782 assert.equal(terminalMember.payload.runId, null);
2783 assert.equal(terminalMember.payload.childThreadId, memberThreadId);
2784 assert.isTrue(
2785 events.some(
2786 (event) =>
2787 event.type === "node.updated" &&
2788 event.payload.id === runningMemberRoot.id &&
2789 event.payload.status === "interrupted",
2790 ),
2791 );
2792 assert.isFalse(
2793 events.some(
2794 (event) => event.type === "subagent.updated" && event.payload.id === foreignSubagent.id,
2795 ),
2796 );
2797 assert.isFalse(
2798 events.some(
2799 (event) => event.type === "node.updated" && event.payload.id === foreignChildNode.id,
2800 ),
2801 );
2802 assert.isFalse(
2803 events.some(
2804 (event) =>
2805 event.type === "turn-item.updated" && event.payload.id === foreignChildTurnItem.id,
2806 ),
2807 );
2808 assert.isDefined(terminalSubagent);
2809 assert.equal(terminalSubagent.payload.status, "interrupted");
2810 assert.equal(terminalSubagent.payload.childThreadId, childThreadId);
2811 assert.equal(terminalSubagent.payload.result, null);
2812 assert.isNotNull(terminalSubagent.payload.completedAt);
2813
2814 assert.lengthOf(turnItemEvents, 1);
2815 const terminalTurnItem = turnItemEvents[0];
2816 assert.isDefined(terminalTurnItem);
2817 assert.equal(terminalTurnItem.payload.status, "interrupted");
2818 assert.equal(terminalTurnItem.payload.childThreadId, childThreadId);
2819 assert.equal(terminalTurnItem.payload.result, null);
2820
2821 const terminalChildTurnItems = events.flatMap((event) =>
2822 event.type === "turn-item.updated" &&
2823 event.payload.id === runningChildTurnItem.id &&
2824 event.payload.status === "interrupted"
2825 ? [event]
2826 : [],
2827 );
2828 assert.lengthOf(terminalChildTurnItems, 1);
2829 const terminalChildTurnItem = terminalChildTurnItems[0];
2830 assert.isDefined(terminalChildTurnItem);
2831 assert.equal(terminalChildTurnItem.threadId, childThreadId);
2832 assert.equal(terminalChildTurnItem.payload.threadId, childThreadId);
2833 assert.equal(terminalChildTurnItem.payload.runId, null);
2834 assert.equal(terminalChildTurnItem.payload.type, "command_execution");
2835 assert.isFalse(
2836 events.some(
2837 (event) =>
2838 event.type === "turn-item.updated" &&
2839 event.payload.id === unrelatedChildTurnItem.id &&
2840 event.payload.status === "interrupted",
2841 ),
2842 "related but unlinked child turn items must not cascade",
2843 );
2844 assert.isFalse(
2845 events.some(
2846 (event) =>
2847 event.type === "turn-item.updated" &&
2848 event.payload.id === suppressedChildAssistantTurnItem.id,
2849 ),
2850 "suppressed streaming child items must not be created by the cascade",
2851 );
2852
2853 assert.lengthOf(nodeEvents, 2);
2854 assert.isTrue(
2855 nodeEvents.some(
2856 (event) => event.payload.id === runningNode.id && event.payload.kind === "subagent",
2857 ),
2858 );
2859 assert.isFalse(
2860 events.some(
2861 (event) =>
2862 event.type === "node.updated" &&
2863 event.payload.id === unrelatedChildNode.id &&
2864 event.payload.status === "interrupted",
2865 ),
2866 "owned child threads without a live run-owned subagent link must not cascade",
2867 );
2868 assert.isTrue(
2869 nodeEvents.some(
2870 (event) =>
2871 event.payload.id === runningChildNode.id &&
2872 event.runId === (runningChildNode.runId ?? ids.runId) &&
2873 event.payload.threadId === childThreadId &&
2874 event.payload.kind === "root_turn",
2875 ),
2876 );
2877
2878 const cascadeIndexes = events.flatMap((event, index) => {
2879 if (event.type === "subagent.updated" && event.payload.status === "interrupted") {
2880 return [index];
2881 }
2882 if (
2883 event.type === "turn-item.updated" &&
2884 event.payload.status === "interrupted" &&
2885 (event.payload.type === "subagent" || event.payload.id === runningChildTurnItem.id)
2886 ) {
2887 return [index];
2888 }
2889 if (event.type === "node.updated" && event.payload.status === "interrupted") {
2890 return event.payload.id === runningNode.id || event.payload.id === runningChildNode.id
2891 ? [index]
2892 : [];
2893 }
2894 return [];
2895 });
2896 assert.isTrue(
2897 cascadeIndexes.every((index) => index < runUpdatedIndex),
2898 "subagent cascade must precede run.updated",
2899 );
2900 assert.isFalse(
2901 events.some((event) => {
2902 if (event.type === "subagent.updated") {
2903 return event.payload.status === "completed";
2904 }
2905 if (event.type === "turn-item.updated" && event.payload.type === "subagent") {
2906 return event.payload.status === "completed";
2907 }
2908 return false;
2909 }),
2910 "late provider completion must not reopen cascaded subagent rows",
2911 );
2912 assert.isFalse(
2913 (yield* Ref.get(ingested)).some(
2914 (event) =>
2915 (event.type === "subagent.updated" && event.subagent.status === "completed") ||
2916 (event.type === "turn_item.updated" && event.turnItem.status === "completed") ||
2917 (event.type === "node.updated" && event.node.status === "completed"),
2918 ),
2919 "late provider completion must not be ingested after interrupt",
2920 );
2921 }),
2922 );
2923
2924 it.effect("settles native subagents when a workspace handoff completes the root run", () =>
2925 Effect.gen(function* () {
2926 const ids = backgroundScenarioIds("subagent-workspace-handoff");
2927 const providerInstanceId = ProviderInstanceId.make("codex");
2928 const written = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);
2929 const ingestionDone = yield* Deferred.make<void>();
2930 const layerTest = RunExecutionService.layer.pipe(
2931 Layer.provide(
2932 Layer.mergeAll(
2933 McpAppModelContext.layerEmpty,
2934 PreviewAgentActivity.layer,
2935 Layer.mock(CheckpointService.CheckpointServiceV2)({
2936 captureBaseline: () => Effect.void,
2937 }),
2938 layerTurnItemPositionsTest,
2939 mockEventSink({
2940 write: () => Effect.succeed([]),
2941 writeWithEffects: (input) =>
2942 Ref.update(written, (current) => [...current, ...input.events]).pipe(Effect.as([])),
2943 writeIfRunCurrent: (input) =>
2944 Ref.update(written, (current) => [...current, ...input.events]).pipe(
2945 Effect.as({ committed: true, storedEvents: [] }),
2946 ),
2947 }),
2948 IdAllocator.layer,
2949 ThreadCommandExecutor.layer,
2950 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
2951 ingestNormalized: () => Effect.succeed([]),
2952 }),
2953 ServerSettings.layerTest(),
2954 ),
2955 ),
2956 );
2957 const fixture = {
2958 ids,
2959 providerInstanceId,
2960 childThreadId: ids.childThreadId,
2961 driver,
2962 status: "running",
2963 } as const;
2964 const runningSubagent = makeRunOwnedSubagentFixture(fixture);
2965 const runningTurnItem = makeRunOwnedSubagentTurnItemFixture(fixture);
2966 const runningNode = makeRunOwnedSubagentNodeFixture({ ids, status: "running" });
2967 const runningChildNode = makeRunOwnedSubagentChildNodeFixture({ ids, status: "running" });
2968
2969 yield* Effect.gen(function* () {
2970 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
2971 yield* runExecution.startRootRun({
2972 commandId: CommandId.make("command:subagent-workspace-handoff"),
2973 appThread: rootAppThread(ids.threadId),
2974 providerSessionId: ProviderSessionId.make("session:subagent-workspace-handoff"),
2975 session: {
2976 events: Stream.empty,
2977 subscribeEvents: Effect.succeed({
2978 // The provider event stream stays open: only settled children may end ingestion.
2979 events: Stream.concat(
2980 Stream.fromIterable([
2981 childThreadCreatedEvent(ids),
2982 { type: "subagent.updated", driver, subagent: runningSubagent },
2983 { type: "node.updated", driver, node: runningNode },
2984 { type: "node.updated", driver, node: runningChildNode },
2985 { type: "turn_item.updated", driver, turnItem: runningTurnItem },
2986 rootTerminalEvent(ids, "interrupted"),
2987 ] satisfies ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>),
2988 Stream.never,
2989 ),
2990 close: Deferred.succeed(ingestionDone, undefined),
2991 }),
2992 startTurn: () => Effect.void,
2993 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
2994 run: {
2995 id: ids.runId,
2996 threadId: ids.threadId,
2997 ordinal: 1,
2998 providerInstanceId,
2999 } as OrchestrationV2Run,
3000 rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode,
3001 checkpointScope: {
3002 id: CheckpointScopeId.make("checkpoint-scope:subagent-workspace-handoff"),
3003 } as OrchestrationV2CheckpointScope,
3004 providerThread: { id: ids.providerThreadId, driver } as OrchestrationV2ProviderThread,
3005 attempt: {
3006 id: ids.attemptId,
3007 providerTurnId: ids.rootProviderTurnId,
3008 } as OrchestrationV2RunAttempt,
3009 attemptId: ids.attemptId,
3010 providerTurnOrdinal: 1,
3011 shouldFinalizeRun: () => Effect.succeed(true),
3012 hasWorkspaceMoved: () => Effect.succeed(true),
3013 message: {
3014 messageId: MessageId.make("message:subagent-workspace-handoff:user"),
3015 text: "Spawn a subagent then move to a worktree.",
3016 attachments: [],
3017 createdBy: "user",
3018 creationSource: "web",
3019 },
3020 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
3021 runtimePolicy: {
3022 runtimeMode: "full-access",
3023 interactionMode: "default",
3024 cwd: process.cwd(),
3025 approvalPolicy: "never",
3026 sandboxPolicy: {
3027 type: "readOnly",
3028 access: { type: "fullAccess" },
3029 networkAccess: false,
3030 },
3031 },
3032 });
3033 }).pipe(Effect.provide(layerTest));
3034
3035 const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds"));
3036 assert.isTrue(Option.isSome(closed), "settled children must end event ingestion");
3037
3038 const events = yield* Ref.get(written);
3039 const statusOf = (type: OrchestrationV2DomainEvent["type"], id: string) =>
3040 events.flatMap((event) => {
3041 if (event.type !== type) return [];
3042 const payload = (
3043 event as { readonly payload: { readonly id: string; readonly status: string } }
3044 ).payload;
3045 return payload.id === id ? [payload.status] : [];
3046 });
3047 assert.deepEqual(statusOf("subagent.updated", runningSubagent.id), ["interrupted"]);
3048 assert.deepEqual(statusOf("turn-item.updated", runningTurnItem.id), ["interrupted"]);
3049 assert.deepEqual(statusOf("node.updated", runningNode.id), ["interrupted"]);
3050 assert.deepEqual(statusOf("node.updated", runningChildNode.id), ["interrupted"]);
3051 assert.deepEqual(statusOf("run.updated", ids.runId), ["waiting"]);
3052 assert.deepEqual(statusOf("run-attempt.updated", ids.attemptId), ["completed"]);
3053 assert.isFalse(
3054 events.some(
3055 (event) =>
3056 event.type === "turn-item.updated" && event.payload.type === "run_interrupt_result",
3057 ),
3058 );
3059 }),
3060 );
3061
3062 it.effect("omits interrupt results and subagent cascade for a superseded attempt", () =>
3063 Effect.gen(function* () {
3064 const { written, observed } = yield* captureRootRunTermination({
3065 key: "steer-supersede",
3066 shouldFinalizeRun: () => Effect.succeed(false),
3067 seedOpenSubagent: true,
3068 });
3069 assert.deepEqual(
3070 written.map((item) => item.type),
3071 [],
3072 );
3073 assert.deepEqual(observed, []);
3074 }),
3075 );
3076
3077 it.effect("emits run_interrupt_result when superseded attempt still has a hard-stop request", () =>
3078 Effect.gen(function* () {
3079 const { written, observed } = yield* captureRootRunTermination({
3080 key: "stop-then-steer-supersede",
3081 shouldFinalizeRun: () => Effect.succeed(false),
3082 hasUnpairedRunInterruptRequest: () => Effect.succeed(true),
3083 });
3084 assert.deepEqual(
3085 written.map((item) => item.type),
3086 ["run_interrupt_result"],
3087 );
3088 assert.deepEqual(observed, ["pull-requests-refreshed"]);
3089 const ids = backgroundScenarioIds("stop-then-steer-supersede");
3090 const expectedRequestId = yield* Effect.gen(function* () {
3091 const idAllocator = yield* IdAllocator.IdAllocatorV2;
3092 return idAllocator.derive.runSignalTurnItem({
3093 runId: ids.runId,
3094 signal: "interrupt-request",
3095 });
3096 }).pipe(Effect.provide(IdAllocator.layer));
3097 assert.equal(written[0]?.parentItemId, expectedRequestId);
3098 }),
3099 );
3100
3101 it.effect("keeps a Stop that commits while a workspace handoff completes the run", () =>
3102 Effect.gen(function* () {
3103 const stopCommitted = yield* Ref.make(false);
3104 const { written, finalEvents, committedEffects } = yield* captureRootRunTermination({
3105 key: "stop-races-workspace-handoff",
3106 shouldFinalizeRun: () => Effect.succeed(true),
3107 // Stop commits after the handoff is detected and before the terminal write.
3108 hasWorkspaceMoved: () => Ref.set(stopCommitted, true).pipe(Effect.as(true)),
3109 // The sink refuses a guarded write once the Stop request is stored.
3110 beforeTerminalWrite: (request) =>
3111 Ref.get(stopCommitted).pipe(
3112 Effect.map((stopped) => !(stopped && request.unlessInterruptRequested !== undefined)),
3113 ),
3114 });
3115 const statuses = (type: OrchestrationV2DomainEvent["type"]) =>
3116 finalEvents.flatMap((event) =>
3117 event.type === type ? [(event.payload as { readonly status: string }).status] : [],
3118 );
3119 assert.deepEqual(statuses("run.updated"), ["interrupted"]);
3120 assert.deepEqual(statuses("run-attempt.updated"), ["interrupted"]);
3121 assert.deepEqual(
3122 written.map((item) => item.type),
3123 ["run_interrupt_result"],
3124 );
3125 assert.deepEqual(
3126 committedEffects.map((effect) => effect.request.type),
3127 ["checkpoint.capture"],
3128 );
3129 }),
3130 );
3131
3132 it.effect("does not overwrite Stop when ownership changes after the finalization read", () =>
3133 Effect.gen(function* () {
3134 const { written, observed, submittedEffects, committedEffects } =
3135 yield* captureRootRunTermination({
3136 key: "stop-wins-finalization-gap",
3137 shouldFinalizeRun: () => Effect.succeed(true),
3138 rejectTerminalWrite: true,
3139 });
3140 assert.deepEqual(written, []);
3141 assert.deepEqual(observed, []);
3142 assert.deepEqual(
3143 submittedEffects.map((effect) => effect.request.type),
3144 ["checkpoint.capture"],
3145 );
3146 assert.deepEqual(committedEffects, []);
3147 }),
3148 );
3149
3150 it.effect.each([
3151 { providerReason: undefined, requestReason: undefined, expected: "Stopped by the provider" },
3152 {
3153 providerReason: "preCompact hook (project): Save a checkpoint.",
3154 requestReason: undefined,
3155 expected: "preCompact hook (project): Save a checkpoint.",
3156 },
3157 {
3158 providerReason: "preCompact hook (project): Save a checkpoint.",
3159 requestReason: "Stopped by the user",
3160 expected: "Stopped by the user",
3161 },
3162 ])(
3163 "emits run_interrupt_result with $expected when finalizing the active attempt",
3164 ({ providerReason, requestReason, expected }) =>
3165 Effect.gen(function* () {
3166 const { written, observed, committedEffects } = yield* captureRootRunTermination({
3167 key: "hard-stop",
3168 shouldFinalizeRun: () => Effect.succeed(true),
3169 loadRunInterruptReason: () => Effect.succeed(requestReason),
3170 events: (ids) =>
3171 Stream.make({ ...rootTerminalEvent(ids, "interrupted"), reason: providerReason }),
3172 });
3173 assert.deepEqual(
3174 written.map((item) => item.type),
3175 ["run_interrupt_result"],
3176 );
3177 assert.deepEqual(observed, ["run:interrupted", "pull-requests-refreshed"]);
3178 assert.equal(
3179 written[0]?.type === "run_interrupt_result" ? written[0].message : undefined,
3180 expected,
3181 );
3182 assert.deepEqual(
3183 committedEffects.map((effect) => effect.request.type),
3184 ["checkpoint.capture"],
3185 );
3186 }),
3187 );
3188
3189 it.effect("keeps a provider's stop reason on a completed run", () =>
3190 Effect.gen(function* () {
3191 const reason = "sessionStart hook (user): Plugin guard blocked this session.";
3192 const { written } = yield* captureRootRunTermination({
3193 key: "completed-stop-reason",
3194 shouldFinalizeRun: () => Effect.succeed(true),
3195 events: (ids) => Stream.make({ ...rootTerminalEvent(ids, "completed"), reason }),
3196 });
3197 assert.deepEqual(
3198 written.map((item) => (item.type === "system_notice" ? item.message : item.type)),
3199 [reason],
3200 );
3201 }),
3202 );
3203
3204 it.effect("records a finished run as failed when its ownership check cannot be read", () =>
3205 Effect.gen(function* () {
3206 const { observed } = yield* captureRootRunTermination({
3207 key: "finalize-guard-read-failure",
3208 shouldFinalizeRun: () =>
3209 Effect.fail(
3210 new ProjectionStore.ProjectionStoreReadError({
3211 threadId: ThreadId.make("thread:finalize-guard-read-failure"),
3212 cause: "database unavailable",
3213 }),
3214 ),
3215 events: (ids) => Stream.make(rootTerminalEvent(ids, "completed")),
3216 });
3217 assert.include(observed, "run:failed");
3218 }),
3219 );
3220
3221 it.effect("does not refresh pull requests for auxiliary or stale provider terminals", () =>
3222 Effect.gen(function* () {
3223 const { observed } = yield* captureRootRunTermination({
3224 key: "pull-request-refresh:auxiliary",
3225 shouldFinalizeRun: () => Effect.succeed(true),
3226 events: (ids) =>
3227 Stream.fromIterable([
3228 rootTerminalEvent(
3229 { ...ids, rootProviderTurnId: ProviderTurnId.make("turn:child") },
3230 "completed",
3231 ),
3232 rootTerminalEvent(
3233 { ...ids, rootProviderTurnId: ProviderTurnId.make("turn:previous") },
3234 "interrupted",
3235 ),
3236 rootTerminalEvent(ids, "cancelled"),
3237 ]),
3238 });
3239 assert.deepEqual(observed, ["run:cancelled", "pull-requests-refreshed"]);
3240 }),
3241 );
3242
3243 it.effect("refreshes pull requests after a provider stream exits with an error", () =>
3244 Effect.gen(function* () {
3245 const { written, observed } = yield* captureRootRunTermination({
3246 key: "pull-request-refresh:stream-error",
3247 shouldFinalizeRun: () => Effect.succeed(true),
3248 events: () =>
3249 Stream.fail(
3250 new ProviderAdapter.ProviderAdapterEventStreamError({
3251 driver,
3252 providerSessionId: ProviderSessionId.make("session:exited"),
3253 cause: "provider process exited",
3254 }),
3255 ),
3256 });
3257 assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3258 assert.deepEqual(
3259 written.map((item) => item.type),
3260 ["error"],
3261 );
3262 const error = written.find((item) => item.type === "error");
3263 assert.include(error?.failure.message ?? "", "provider event stream closed unexpectedly");
3264 }),
3265 );
3266
3267 it.effect("records a provider's input rejection at turn start in the terminal error", () =>
3268 Effect.gen(function* () {
3269 const rejection = "goal objective must be at most 4000 characters";
3270 const { written, observed } = yield* captureRootRunTermination({
3271 key: "start-failure:input-rejected",
3272 shouldFinalizeRun: () => Effect.succeed(true),
3273 events: () => Stream.never,
3274 afterStart: (closed) => Deferred.succeed(closed, undefined),
3275 startTurn: (turnInput) =>
3276 Effect.fail(
3277 new ProviderAdapter.ProviderAdapterTurnStartError({
3278 driver,
3279 threadId: turnInput.threadId,
3280 providerThreadId: turnInput.providerThread.id,
3281 runId: turnInput.runId,
3282 cause: new CodexAppServerRequestError({
3283 code: -32600,
3284 errorMessage: rejection,
3285 method: "thread/goal/set",
3286 }),
3287 }),
3288 ),
3289 });
3290 assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3291 const error = written.find((item) => item.type === "error");
3292 assert.include(error?.failure.message ?? "", rejection);
3293 }),
3294 );
3295
3296 it.effect("tells the user why /compact failed on a provider without compaction", () =>
3297 Effect.gen(function* () {
3298 const { written } = yield* captureRootRunTermination({
3299 key: "start-failure:compact-unsupported",
3300 shouldFinalizeRun: () => Effect.succeed(true),
3301 events: () => Stream.never,
3302 afterStart: (closed) => Deferred.succeed(closed, undefined),
3303 messageText: "/compact",
3304 });
3305 const error = written.find((item) => item.type === "error");
3306 assert.include(error?.failure.message ?? "", "does not support /compact");
3307 }),
3308 );
3309
3310 it.effect.each(["failure", "defect", "interrupt-failure", "ownership-lost"] as const)(
3311 "stops the live provider and background work before settling an ingestion %s",
3312 (failureMode) =>
3313 Effect.gen(function* () {
3314 const started = yield* Deferred.make<void>();
3315 const stopped = yield* Deferred.make<void>();
3316 const backgroundWorkRunning = yield* Ref.make(true);
3317 const backgroundWorkAtFinalization = yield* Ref.make<boolean | null>(null);
3318 const now = yield* DateTime.now;
3319 const interrupts = yield* Ref.make<
3320 ReadonlyArray<ProviderAdapter.ProviderAdapterV2InterruptInput>
3321 >([]);
3322 const ids = backgroundScenarioIds(`ingestion:${failureMode}`);
3323 const { written, observed, finalEvents } = yield* captureRootRunTermination({
3324 key: `ingestion:${failureMode}`,
3325 initialProviderTurnId: null,
3326 rejectTerminalWrite: failureMode === "ownership-lost",
3327 shouldFinalizeRun: () => Effect.succeed(true),
3328 startTurn: () =>
3329 Deferred.succeed(started, undefined).pipe(Effect.andThen(Deferred.await(stopped))),
3330 events: () =>
3331 Stream.unwrap(
3332 Deferred.await(started).pipe(
3333 Effect.as(
3334 Stream.make({
3335 type: "provider_turn.updated" as const,
3336 driver,
3337 providerTurn: {
3338 id: ids.rootProviderTurnId,
3339 providerThreadId: ids.providerThreadId,
3340 nodeId: ids.rootNodeId,
3341 runAttemptId: ids.attemptId,
3342 nativeTurnRef: null,
3343 ordinal: 1,
3344 status: "running" as const,
3345 startedAt: now,
3346 completedAt: null,
3347 },
3348 }),
3349 ),
3350 ),
3351 ),
3352 ingestNormalized: () => {
3353 const error = new ProviderEventIngestor.ProviderEventPublishError({
3354 providerSessionId: ProviderSessionId.make(`session:${failureMode}`),
3355 eventCount: 1,
3356 cause: toPersistenceSqlError("OrchestrationEventStore.appendAgentEvents:insert")(
3357 new SqlError({
3358 reason: classifySqliteError(
3359 Object.assign(new Error("unable to open database file"), { errcode: 14 }),
3360 ),
3361 }),
3362 ),
3363 });
3364 return failureMode === "defect" ? Effect.die(error) : Effect.fail(error);
3365 },
3366 interruptTurn: (input) =>
3367 Ref.update(interrupts, (current) => [...current, input]).pipe(
3368 Effect.andThen(
3369 input.requestRuntimeRestart === true && failureMode !== "interrupt-failure"
3370 ? Ref.set(backgroundWorkRunning, false)
3371 : Effect.void,
3372 ),
3373 Effect.andThen(Deferred.succeed(stopped, undefined)),
3374 Effect.andThen(
3375 failureMode === "interrupt-failure"
3376 ? Effect.fail(
3377 new ProviderAdapter.ProviderAdapterInterruptError({
3378 driver,
3379 providerThreadId: input.providerThread.id,
3380 providerTurnId: input.providerTurnId,
3381 cause: "provider already exited",
3382 }),
3383 )
3384 : Effect.void,
3385 ),
3386 ),
3387 beforeTerminalWrite: () =>
3388 Effect.gen(function* () {
3389 const calls = yield* Ref.get(interrupts);
3390 assert.equal(calls.length, 1);
3391 assert.equal(calls[0]?.providerTurnId, ids.rootProviderTurnId);
3392 assert.equal(calls[0]?.providerThread.id, ids.providerThreadId);
3393 yield* Ref.set(backgroundWorkAtFinalization, yield* Ref.get(backgroundWorkRunning));
3394 return true;
3395 }),
3396 });
3397 if (failureMode === "ownership-lost") {
3398 assert.deepEqual(finalEvents, []);
3399 assert.deepEqual(observed, []);
3400 } else {
3401 assert.equal(
3402 yield* Ref.get(backgroundWorkAtFinalization),
3403 failureMode === "interrupt-failure",
3404 );
3405 assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3406 const error = written.find((item) => item.type === "error");
3407 assert.include(error?.failure.message ?? "", "(SQLITE_CANTOPEN)");
3408 const repaired = finalEvents.find((event) => event.type === "provider-turn.updated");
3409 if (failureMode === "interrupt-failure") {
3410 assert.isUndefined(repaired);
3411 } else {
3412 assert.equal(repaired?.payload.id, ids.rootProviderTurnId);
3413 assert.equal(repaired?.payload.status, "interrupted");
3414 assert.isNotNull(repaired?.payload.completedAt);
3415 }
3416 }
3417 }),
3418 );
3419
3420 it.effect.each([
3421 { status: "completed", failure: "later-stream", interrupt: "succeeds" },
3422 { status: "failed", failure: "later-stream", interrupt: "succeeds" },
3423 { status: "completed", failure: "terminal-ingest", interrupt: "succeeds" },
3424 { status: "completed", failure: "terminal-ingest", interrupt: "fails" },
3425 ] as const)(
3426 "stores a $status root provider turn snapshot after a $failure failure only if the interrupt succeeds (it $interrupt)",
3427 ({ status, failure, interrupt }) =>
3428 Effect.gen(function* () {
3429 const now = yield* DateTime.now;
3430 const terminalAt = DateTime.subtract(now, { seconds: 1 });
3431 const key = `terminal-snapshot:${status}:${failure}:${interrupt}`;
3432 const durableTurns = yield* Ref.make<
3433 ReadonlyMap<ProviderTurnId, OrchestrationV2ProviderTurn>
3434 >(new Map());
3435 const persist = (turn: OrchestrationV2ProviderTurn) =>
3436 Ref.update(durableTurns, (current) => new Map(current).set(turn.id, turn));
3437 const interrupts = yield* Ref.make<
3438 ReadonlyArray<ProviderAdapter.ProviderAdapterV2InterruptInput>
3439 >([]);
3440 const ids = backgroundScenarioIds(key);
3441 const rootTurn = (
3442 turnStatus: OrchestrationV2ProviderTurn["status"],
3443 completedAt: DateTime.Utc | null,
3444 ) => ({
3445 type: "provider_turn.updated" as const,
3446 driver,
3447 providerTurn: {
3448 id: ids.rootProviderTurnId,
3449 providerThreadId: ids.providerThreadId,
3450 nodeId: ids.rootNodeId,
3451 runAttemptId: ids.attemptId,
3452 nativeTurnRef: null,
3453 ordinal: 1,
3454 status: turnStatus,
3455 startedAt: terminalAt,
3456 completedAt,
3457 },
3458 });
3459 const { observed, finalEvents } = yield* captureRootRunTermination({
3460 key,
3461 shouldFinalizeRun: () => Effect.succeed(true),
3462 events: () =>
3463 Stream.make(rootTurn("running", null), rootTurn(status, terminalAt)).pipe(
3464 Stream.concat(
3465 failure === "later-stream"
3466 ? Stream.fail(
3467 new ProviderAdapter.ProviderAdapterEventStreamError({
3468 driver,
3469 providerSessionId: ProviderSessionId.make(`session:${key}`),
3470 cause: "provider process exited",
3471 }),
3472 )
3473 : Stream.empty,
3474 ),
3475 ),
3476 ingestNormalized: ({ event }) =>
3477 event.type !== "provider_turn.updated"
3478 ? Effect.succeed([])
3479 : failure === "terminal-ingest" && event.providerTurn.status !== "running"
3480 ? Effect.fail(
3481 new ProviderEventIngestor.ProviderEventPublishError({
3482 providerSessionId: ProviderSessionId.make(`session:${key}`),
3483 eventCount: 1,
3484 cause: new Error("SQLITE_FULL: database or disk is full"),
3485 }),
3486 )
3487 : persist(event.providerTurn).pipe(Effect.as([])),
3488 interruptTurn: (input) =>
3489 Ref.update(interrupts, (current) => [...current, input]).pipe(
3490 Effect.andThen(
3491 interrupt === "fails"
3492 ? Effect.fail(
3493 new ProviderAdapter.ProviderAdapterInterruptError({
3494 driver,
3495 providerThreadId: input.providerThread.id,
3496 providerTurnId: input.providerTurnId,
3497 cause: "provider already exited",
3498 }),
3499 )
3500 : Effect.void,
3501 ),
3502 ),
3503 beforeTerminalWrite: (request) =>
3504 Effect.forEach(request.events, (event) =>
3505 event.type === "provider-turn.updated" ? persist(event.payload) : Effect.void,
3506 ).pipe(Effect.as(true)),
3507 });
3508 const calls = yield* Ref.get(interrupts);
3509 assert.equal(calls.length, 1);
3510 assert.isTrue(calls[0]?.requestRuntimeRestart);
3511 assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3512 assert.isTrue(finalEvents.some((event) => event.type === "run.updated"));
3513 const durable = (yield* Ref.get(durableTurns)).get(ids.rootProviderTurnId);
3514 // A failed interrupt keeps the stored turn running for the next start to retry.
3515 assert.equal(durable?.status, interrupt === "fails" ? "running" : status);
3516 assert.deepEqual(durable?.completedAt, interrupt === "fails" ? null : terminalAt);
3517 }),
3518 );
3519
3520 it.effect("does not interrupt an ended provider when its terminal write fails", () =>
3521 Effect.gen(function* () {
3522 const interrupts = yield* Ref.make(0);
3523 const writes = yield* Ref.make(0);
3524 const { observed } = yield* captureRootRunTermination({
3525 key: "terminal-write-failure",
3526 shouldFinalizeRun: () => Effect.succeed(true),
3527 events: (ids) => Stream.make(rootTerminalEvent(ids, "completed")),
3528 interruptTurn: () => Ref.update(interrupts, (count) => count + 1),
3529 beforeTerminalWrite: () =>
3530 Effect.gen(function* () {
3531 if ((yield* Ref.updateAndGet(writes, (count) => count + 1)) === 1) {
3532 return yield* new EventSink.EventSinkWriteError({ eventCount: 1 });
3533 }
3534 return true;
3535 }),
3536 });
3537 assert.equal(yield* Ref.get(interrupts), 0);
3538 assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3539 }),
3540 );
3541
3542 it.effect("ends a running root command when the provider stream fails mid-turn", () =>
3543 Effect.gen(function* () {
3544 const now = yield* DateTime.now;
3545 const { written, observed } = yield* captureRootRunTermination({
3546 key: "stream-error-running-command",
3547 shouldFinalizeRun: () => Effect.succeed(true),
3548 events: (ids) =>
3549 Stream.make(
3550 {
3551 type: "provider_turn.updated" as const,
3552 driver,
3553 providerTurn: {
3554 id: ids.rootProviderTurnId,
3555 providerThreadId: ids.providerThreadId,
3556 nodeId: ids.rootNodeId,
3557 runAttemptId: ids.attemptId,
3558 nativeTurnRef: null,
3559 ordinal: 1,
3560 status: "running" as const,
3561 startedAt: now,
3562 completedAt: null,
3563 },
3564 },
3565 backgroundTurnItemEvent(ids, "command_execution", "running", 1),
3566 ).pipe(
3567 Stream.concat(
3568 Stream.fail(
3569 new ProviderAdapter.ProviderAdapterEventStreamError({
3570 driver,
3571 providerSessionId: ProviderSessionId.make("session:stream-error-running-command"),
3572 cause: "provider process exited",
3573 }),
3574 ),
3575 ),
3576 ),
3577 // The dead process cannot take the interrupt either.
3578 interruptTurn: (input) =>
3579 Effect.fail(
3580 new ProviderAdapter.ProviderAdapterInterruptError({
3581 driver,
3582 providerThreadId: input.providerThread.id,
3583 providerTurnId: input.providerTurnId,
3584 cause: "provider already exited",
3585 }),
3586 ),
3587 });
3588 assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3589 const ids = backgroundScenarioIds("stream-error-running-command");
3590 const endedCommand = written.find(
3591 (item) => item.id === ids.itemId && item.status !== "running",
3592 );
3593 assert.isDefined(endedCommand);
3594 }),
3595 );
3596
3597 it.effect("keeps a running root command open when a write fails under a live provider", () =>
3598 Effect.gen(function* () {
3599 const key = "write-error-running-command";
3600 const ids = backgroundScenarioIds(key);
3601 const secondItemId = TurnItemId.make(`${ids.itemId}:second`);
3602 const { written, observed } = yield* captureRootRunTermination({
3603 key,
3604 shouldFinalizeRun: () => Effect.succeed(true),
3605 events: () =>
3606 Stream.make(
3607 backgroundTurnItemEvent(ids, "command_execution", "running", 1),
3608 backgroundTurnItemEvent(ids, "command_execution", "running", 2, secondItemId),
3609 ),
3610 ingestNormalized: ({ event }) =>
3611 event.type === "turn_item.updated" && event.turnItem.id === secondItemId
3612 ? Effect.fail(
3613 new ProviderEventIngestor.ProviderEventPublishError({
3614 providerSessionId: ProviderSessionId.make(`session:${key}`),
3615 eventCount: 1,
3616 cause: new Error("SQLITE_FULL: database or disk is full"),
3617 }),
3618 )
3619 : Effect.succeed([]),
3620 });
3621 assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3622 assert.isUndefined(written.find((item) => item.id === ids.itemId));
3623 }),
3624 );
3625
3626 it.effect.each([
3627 { ended: true },
3628 // T3 ending the stream proves no dead runtime; Stop settles the work it releases.
3629 { ended: false, raisedByT3: true },
3630 // A Stop that already settled the subagent keeps its result.
3631 { ended: false, stopped: true },
3632 ] as const)(
3633 "ends a subagent that outlived its root when the provider stream fails (%o)",
3634 ({ ended, raisedByT3, stopped }) =>
3635 Effect.gen(function* () {
3636 const providerInstanceId = ProviderInstanceId.make("codex");
3637 const { finalEvents } = yield* captureRootRunTermination({
3638 key: "stream-error-outlived-subagent",
3639 shouldFinalizeRun: () => Effect.succeed(true),
3640 events: (ids) =>
3641 Stream.make(
3642 {
3643 type: "subagent.updated" as const,
3644 driver,
3645 subagent: makeRunOwnedSubagentFixture({
3646 ids,
3647 providerInstanceId,
3648 childThreadId: ids.childThreadId,
3649 driver,
3650 status: "running",
3651 }),
3652 },
3653 {
3654 type: "turn_item.updated" as const,
3655 driver,
3656 turnItem: makeRunOwnedSubagentTurnItemFixture({
3657 ids,
3658 providerInstanceId,
3659 childThreadId: ids.childThreadId,
3660 driver,
3661 status: "running",
3662 }),
3663 },
3664 rootTerminalEvent(ids, "completed"),
3665 ).pipe(
3666 Stream.concat(
3667 Stream.fail(
3668 new ProviderAdapter.ProviderAdapterEventStreamError({
3669 driver,
3670 providerSessionId: ProviderSessionId.make(
3671 "session:stream-error-outlived-subagent",
3672 ),
3673 ...(raisedByT3 === undefined ? {} : { raisedByT3 }),
3674 cause: "provider process exited",
3675 }),
3676 ),
3677 ),
3678 ),
3679 ...(stopped === true
3680 ? {
3681 settleStored: (work) => ({
3682 ...work,
3683 subagents: work.subagents.map((row) => ({ ...row, status: "interrupted" })),
3684 turnItems: work.turnItems.map((row) => ({ ...row, status: "interrupted" })),
3685 }),
3686 }
3687 : {}),
3688 });
3689 assert.deepEqual(
3690 finalEvents.flatMap((event) =>
3691 event.type === "run.updated" ||
3692 event.type === "subagent.updated" ||
3693 event.type === "turn-item.updated"
3694 ? [`${event.type}:${event.payload.status}`]
3695 : [],
3696 ),
3697 ended
3698 ? ["run.updated:waiting", "subagent.updated:failed", "turn-item.updated:failed"]
3699 : ["run.updated:waiting"],
3700 );
3701 }),
3702 );
3703
3704 it.effect("does not interrupt a provider when ingestion fails before the root turn is known", () =>
3705 Effect.gen(function* () {
3706 const interrupts = yield* Ref.make(0);
3707 const { observed } = yield* captureRootRunTermination({
3708 key: "unknown-turn-stream-error",
3709 initialProviderTurnId: null,
3710 shouldFinalizeRun: () => Effect.succeed(true),
3711 events: () =>
3712 Stream.fail(
3713 new ProviderAdapter.ProviderAdapterEventStreamError({
3714 driver,
3715 providerSessionId: ProviderSessionId.make("session:unknown-turn"),
3716 cause: "provider process exited",
3717 }),
3718 ),
3719 interruptTurn: () => Ref.update(interrupts, (count) => count + 1),
3720 });
3721 assert.equal(yield* Ref.get(interrupts), 0);
3722 assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3723 }),
3724 );
3725
3726 it.effect("interrupts a provider turn that starts after Stop settled its attempt", () =>
3727 Effect.gen(function* () {
3728 const now = yield* DateTime.now;
3729 const key = "turn-after-stop";
3730 const ids = backgroundScenarioIds(key);
3731 const stopped = yield* Ref.make(false);
3732 const accepted = yield* Deferred.make<void>();
3733 const interrupted = yield* Deferred.make<ProviderAdapter.ProviderAdapterV2InterruptInput>();
3734 yield* captureRootRunTermination({
3735 key,
3736 initialProviderTurnId: null,
3737 shouldFinalizeRun: () => Ref.get(stopped).pipe(Effect.map((value) => !value)),
3738 shouldStartProviderTurn: () => Ref.get(stopped).pipe(Effect.map((value) => !value)),
3739 // Stop commits while the provider accepts the prompt.
3740 startTurn: () =>
3741 Ref.set(stopped, true).pipe(Effect.andThen(Deferred.succeed(accepted, undefined))),
3742 events: () =>
3743 Stream.unwrap(
3744 Deferred.await(accepted).pipe(
3745 Effect.as(
3746 Stream.make({
3747 type: "provider_turn.updated" as const,
3748 driver,
3749 providerTurn: {
3750 id: ids.rootProviderTurnId,
3751 providerThreadId: ids.providerThreadId,
3752 nodeId: ids.rootNodeId,
3753 runAttemptId: ids.attemptId,
3754 nativeTurnRef: null,
3755 ordinal: 1,
3756 status: "running" as const,
3757 startedAt: now,
3758 completedAt: null,
3759 },
3760 }).pipe(
3761 Stream.concat(
3762 Stream.fromEffect(
3763 Deferred.await(interrupted).pipe(
3764 Effect.as(rootTerminalEvent(ids, "interrupted")),
3765 ),
3766 ),
3767 ),
3768 ),
3769 ),
3770 ),
3771 ),
3772 interruptTurn: (input) => Deferred.succeed(interrupted, input).pipe(Effect.asVoid),
3773 });
3774 const call = yield* Deferred.await(interrupted);
3775 assert.equal(call.providerTurnId, ids.rootProviderTurnId);
3776 assert.equal(call.providerThread.id, ids.providerThreadId);
3777 }),
3778 );
3779
3780 // The server browser keeps an agent's tab focused for exactly this span.
3781 it.effect.each(["completed", "failed"] as const)(
3782 "reports the thread executing until its %s run's event consumer ends",
3783 (outcome) =>
3784 Effect.gen(function* () {
3785 const gate = yield* Deferred.make<void>();
3786 const { previewActivityEnded } = yield* captureRootRunTermination({
3787 key: `preview-activity:${outcome}`,
3788 shouldFinalizeRun: () => Effect.succeed(true),
3789 events: (ids) =>
3790 Stream.fromEffect(Deferred.await(gate)).pipe(
3791 Stream.flatMap(() =>
3792 outcome === "completed"
3793 ? Stream.make(rootTerminalEvent(ids, "completed"))
3794 : Stream.fail(
3795 new ProviderAdapter.ProviderAdapterEventStreamError({
3796 driver,
3797 providerSessionId: ProviderSessionId.make("session:exited"),
3798 cause: "provider process exited",
3799 }),
3800 ),
3801 ),
3802 ),
3803 afterStart: () =>
3804 Effect.gen(function* () {
3805 const activity = yield* PreviewAgentActivity.PreviewAgentActivity;
3806 assert.isTrue(
3807 yield* activity.active(ThreadId.make(`thread:preview-activity:${outcome}`)),
3808 );
3809 yield* Deferred.succeed(gate, undefined);
3810 }),
3811 });
3812 assert.isTrue(previewActivityEnded);
3813 }),
3814 );
3815
3816 // Background commands keep the consumer alive for the whole session; only the agent's own
3817 // turn and the children it drives should hold the browser tab focused.
3818 it.effect.each([
3819 { name: "ends once the root turn finishes beside background work", openSubagent: false },
3820 { name: "stays while a run-owned subagent continues", openSubagent: true },
3821 ])("preview activity $name", ({ openSubagent }) =>
3822 Effect.gen(function* () {
3823 const key = `preview-activity:background:${openSubagent}`;
3824 const consumerWaiting = yield* Deferred.make<void>();
3825 const release = yield* Deferred.make<void>();
3826 const providerInstanceId = ProviderInstanceId.make("codex");
3827 yield* captureRootRunTermination({
3828 key,
3829 shouldFinalizeRun: () => Effect.succeed(true),
3830 hasPendingBackgroundWorkForThread: () => Effect.succeed(true),
3831 events: (ids) =>
3832 Stream.concat(
3833 Stream.fromIterable([
3834 ...(openSubagent
3835 ? [
3836 {
3837 type: "subagent.updated",
3838 driver,
3839 subagent: makeRunOwnedSubagentFixture({
3840 ids,
3841 providerInstanceId,
3842 childThreadId: ids.childThreadId,
3843 driver,
3844 status: "running",
3845 }),
3846 } as const,
3847 ]
3848 : []),
3849 rootTerminalEvent(ids, "completed"),
3850 ] satisfies ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>),
3851 Stream.fromEffect(
3852 Deferred.succeed(consumerWaiting, undefined).pipe(
3853 Effect.andThen(Deferred.await(release)),
3854 ),
3855 ).pipe(Stream.drain),
3856 ),
3857 afterStart: (closed) =>
3858 Effect.gen(function* () {
3859 yield* Deferred.await(consumerWaiting);
3860 assert.isFalse(yield* Deferred.isDone(closed));
3861 const activity = yield* PreviewAgentActivity.PreviewAgentActivity;
3862 assert.equal(yield* activity.active(ThreadId.make(`thread:${key}`)), openSubagent);
3863 yield* Deferred.succeed(release, undefined);
3864 }),
3865 });
3866 }),
3867 );
3868
3869 it.effect("fails the run when its event stream ends without a root terminal", () =>
3870 Effect.gen(function* () {
3871 const { observed, finalEvents } = yield* captureRootRunTermination({
3872 key: "stream-end-without-terminal",
3873 shouldFinalizeRun: () => Effect.succeed(true),
3874 events: () => Stream.empty,
3875 });
3876 assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3877 assert.equal(
3878 finalEvents.find((event) => event.type === "provider-thread.updated")?.payload.status,
3879 "idle",
3880 );
3881 }),
3882 );
3883
3884 it.effect("leaves the run to restart recovery when shutdown interrupts its event stream", () =>
3885 Effect.gen(function* () {
3886 const { written, observed } = yield* captureRootRunTermination({
3887 key: "stream-end:shutdown",
3888 shouldFinalizeRun: () => Effect.succeed(true),
3889 events: () => Stream.failCause(Cause.interrupt()),
3890 });
3891 assert.deepEqual(written, []);
3892 assert.deepEqual(observed, []);
3893 }),
3894 );
3895
3896 it.effect("does not start a turn whose run settled during the app context read", () =>
3897 Effect.gen(function* () {
3898 const contextReadStarted = yield* Deferred.make<void>();
3899 const runFailed = yield* Deferred.make<void>();
3900 const current = yield* Ref.make(true);
3901 const providerCalls = yield* Ref.make(0);
3902 const { observed } = yield* captureRootRunTermination({
3903 key: "stream-end:context-read",
3904 shouldStartProviderTurn: () => Ref.get(current),
3905 shouldFinalizeRun: () => Ref.get(current),
3906 events: () => Stream.unwrap(Deferred.await(contextReadStarted).pipe(Effect.as(Stream.empty))),
3907 appContext: Deferred.succeed(contextReadStarted, undefined).pipe(
3908 Effect.andThen(Deferred.await(runFailed)),
3909 Effect.as([]),
3910 ),
3911 refreshAfterTurn: Ref.set(current, false).pipe(
3912 Effect.andThen(Deferred.succeed(runFailed, undefined)),
3913 ),
3914 startTurn: () => Ref.update(providerCalls, (count) => count + 1),
3915 });
3916 assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3917 assert.equal(yield* Ref.get(providerCalls), 0);
3918 }),
3919 );
3920
3921 it.effect("keeps completed runs completed when pull request refresh fails", () =>
3922 Effect.gen(function* () {
3923 const { observed } = yield* captureRootRunTermination({
3924 key: "pull-request-refresh:refresh-failure",
3925 shouldFinalizeRun: () => Effect.succeed(true),
3926 events: (ids) => Stream.make(rootTerminalEvent(ids, "completed")),
3927 refreshAfterTurn: Effect.die("refresh failed"),
3928 });
3929 assert.deepEqual(observed, ["run:waiting", "pull-requests-refreshed"]);
3930 }),
3931 );
3932
3933 it.effect.each(["completed", "failed"] as const)(
3934 "persists a background-task roster clear that arrives after a %s turn",
3935 (status) =>
3936 Effect.gen(function* () {
3937 const pending = yield* Ref.make(true);
3938 const ingested = yield* Ref.make<Array<ProviderAdapter.ProviderAdapterV2Event>>([]);
3939 yield* captureRootRunTermination({
3940 key: `roster-clear-${status}`,
3941 shouldFinalizeRun: () => Effect.succeed(true),
3942 hasPendingBackgroundWorkForThread: () => Ref.get(pending),
3943 events: (ids) => {
3944 const runningBuild: OrchestrationV2PendingBackgroundTask = {
3945 taskId: "running-build",
3946 kind: "command",
3947 };
3948 const roster = (hasTask: boolean): ProviderAdapter.ProviderAdapterV2Event => {
3949 const pendingBackgroundTasks: ReadonlyArray<OrchestrationV2PendingBackgroundTask> =
3950 hasTask ? [runningBuild] : [];
3951 return {
3952 type: "provider_thread.updated",
3953 driver,
3954 providerThread: {
3955 id: ids.providerThreadId,
3956 driver,
3957 appThreadId: ids.threadId,
3958 pendingBackgroundTasks,
3959 } as OrchestrationV2ProviderThread,
3960 };
3961 };
3962 return Stream.fromIterable([roster(true), rootTerminalEvent(ids, status), roster(false)]);
3963 },
3964 // The adapter clears its live roster before ingestion reaches the queued clear.
3965 beforeTerminalWrite: () => Ref.set(pending, false).pipe(Effect.as(true)),
3966 ingestNormalized: (input) =>
3967 Ref.update(ingested, (all) => [...all, input.event]).pipe(
3968 Effect.as(
3969 input.event.type === "provider_thread.updated"
3970 ? [{} as OrchestrationV2StoredEvent]
3971 : [],
3972 ),
3973 ),
3974 });
3975 const rosters = (yield* Ref.get(ingested)).flatMap((event) =>
3976 event.type === "provider_thread.updated"
3977 ? [event.providerThread.pendingBackgroundTasks]
3978 : [],
3979 );
3980 assert.deepEqual(rosters.at(-1), [], "an ended task must leave the persisted roster");
3981 }),
3982 );
3983
3984 function captureRootRunTermination(input: {
3985 readonly key: string;
3986 readonly shouldFinalizeRun: () => Effect.Effect<boolean, ProjectionStore.ProjectionStoreV2Error>;
3987 readonly rejectTerminalWrite?: boolean;
3988 readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
3989 readonly hasWorkspaceMoved?: () => Effect.Effect<boolean, never>;
3990 readonly loadRunInterruptReason?: () => Effect.Effect<string | undefined, never>;
3991 readonly hasPendingBackgroundWorkForThread?: ProviderAdapter.ProviderAdapterV2SessionRuntime["hasPendingBackgroundWorkForThread"];
3992 readonly seedOpenSubagent?: boolean;
3993 readonly events?: (
3994 ids: BackgroundScenarioIds,
3995 ) => Stream.Stream<
3996 ProviderAdapter.ProviderAdapterV2Event,
3997 ProviderAdapter.ProviderAdapterV2Error
3998 >;
3999 readonly startTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["startTurn"];
4000 readonly interruptTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["interruptTurn"];
4001 readonly messageText?: string;
4002 readonly initialProviderTurnId?: ProviderTurnId | null;
4003 readonly onStartFailure?: (
4004 error: RunExecutionService.RunExecutionServiceV2Error,
4005 ) => Effect.Effect<void>;
4006 readonly refreshAfterTurn?: Effect.Effect<void>;
4007 readonly onSubscriptionClose?: Effect.Effect<void>;
4008 readonly shouldStartProviderTurn?: RunExecutionService.RunExecutionServiceV2StartRootRunInput["shouldStartProviderTurn"];
4009 readonly appContext?: ReturnType<McpAppModelContext.McpAppModelContext["Service"]["forThread"]>;
4010 readonly ingestNormalized?: ProviderEventIngestor.ProviderEventIngestorV2["Service"]["ingestNormalized"];
4011 readonly beforeTerminalWrite?: (
4012 request: Parameters<EventSink.EventSinkV2["Service"]["writeIfRunCurrent"]>[0],
4013 ) => Effect.Effect<boolean, EventSink.EventSinkV2Error>;
4014 readonly afterStart?: (
4015 closed: Deferred.Deferred<void>,
4016 ) => Effect.Effect<void, never, PreviewAgentActivity.PreviewAgentActivity>;
4017 /** Changes the stored rows after the provider reported them, as a concurrent Stop does. */
4018 readonly settleStored?: (
4019 work: RunExecutionService.UnsettledWork,
4020 ) => RunExecutionService.UnsettledWork;
4021 }) {
4022 return Effect.gen(function* () {
4023 const ids = backgroundScenarioIds(input.key);
4024 // The rows the provider's events stored, which the run reads back when its runtime is lost.
4025 const stored = {
4026 subagents: new Map<string, OrchestrationV2Subagent>(),
4027 nodes: new Map<string, OrchestrationV2ExecutionNode>(),
4028 turnItems: new Map<string, OrchestrationV2TurnItem>(),
4029 };
4030 const store = (event: ProviderAdapter.ProviderAdapterV2Event) =>
4031 Effect.sync(() => {
4032 if (event.type === "subagent.updated")
4033 stored.subagents.set(event.subagent.id, event.subagent);
4034 if (event.type === "node.updated") stored.nodes.set(event.node.id, event.node);
4035 if (event.type === "turn_item.updated")
4036 stored.turnItems.set(event.turnItem.id, event.turnItem);
4037 });
4038 const loadUnsettledWork = (threadId: ThreadId) =>
4039 Effect.sync(() => {
4040 const work = {
4041 subagents: [...stored.subagents.values()].filter((row) => row.threadId === threadId),
4042 nodes: [...stored.nodes.values()].filter((row) => row.threadId === threadId),
4043 turnItems: [...stored.turnItems.values()].filter((row) => row.threadId === threadId),
4044 };
4045 return input.settleStored?.(work) ?? work;
4046 });
4047 const providerInstanceId = ProviderInstanceId.make("codex");
4048 const runningSubagent = makeRunOwnedSubagentFixture({
4049 ids,
4050 providerInstanceId,
4051 childThreadId: ids.childThreadId,
4052 driver,
4053 status: "running",
4054 });
4055 const writtenItems = yield* Ref.make<ReadonlyArray<OrchestrationV2TurnItem>>([]);
4056 const finalEvents = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);
4057 const observed = yield* Ref.make<ReadonlyArray<string>>([]);
4058 const submittedEffects = yield* Ref.make<ReadonlyArray<PendingOrchestrationEffectV2>>([]);
4059 const committedEffects = yield* Ref.make<ReadonlyArray<PendingOrchestrationEffectV2>>([]);
4060 const ingestionDone = yield* Deferred.make<void>();
4061 const captureTurnItem = (payload: OrchestrationV2TurnItem) =>
4062 Ref.update(writtenItems, (current) => [...current, payload]);
4063 const captureFinalEvents = (events: ReadonlyArray<OrchestrationV2DomainEvent>) =>
4064 Effect.gen(function* () {
4065 yield* Ref.update(finalEvents, (current) => [...current, ...events]);
4066 for (const event of events) {
4067 if (event.type === "turn-item.updated") {
4068 yield* captureTurnItem(event.payload);
4069 }
4070 if (event.type === "run.updated") {
4071 yield* Ref.update(observed, (current) => [...current, `run:${event.payload.status}`]);
4072 }
4073 }
4074 });
4075 const layerTest = RunExecutionService.layer.pipe(
4076 Layer.provide(
4077 Layer.mergeAll(
4078 Layer.mock(McpAppModelContext.McpAppModelContext)({
4079 forThread: () => input.appContext ?? Effect.succeed([]),
4080 }),
4081 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
4082 layerTurnItemPositionsTest,
4083 mockEventSink({
4084 write: (payload) =>
4085 Effect.gen(function* () {
4086 for (const event of payload.events) {
4087 if (event.type === "turn-item.updated") {
4088 yield* captureTurnItem(event.payload);
4089 }
4090 }
4091 return [];
4092 }),
4093 writeWithEffects: (payload) =>
4094 Effect.gen(function* () {
4095 yield* Ref.update(submittedEffects, (current) => [...current, ...payload.effects]);
4096 yield* Ref.update(committedEffects, (current) => [...current, ...payload.effects]);
4097 yield* captureFinalEvents(payload.events);
4098 return [];
4099 }),
4100 writeIfRunCurrent: (payload) =>
4101 Effect.gen(function* () {
4102 const effects = payload.effects ?? [];
4103 yield* Ref.update(submittedEffects, (current) => [...current, ...effects]);
4104 if (
4105 input.rejectTerminalWrite === true ||
4106 (input.beforeTerminalWrite !== undefined &&
4107 !(yield* input.beforeTerminalWrite(payload)))
4108 ) {
4109 return { committed: false, storedEvents: [] };
4110 }
4111 yield* Ref.update(committedEffects, (current) => [...current, ...effects]);
4112 yield* captureFinalEvents(payload.events);
4113 return { committed: true, storedEvents: [] };
4114 }),
4115 // Committed only against the row that was read, as the real conditional writes are.
4116 writeIfSubagentUnchanged: (payload) =>
4117 stored.subagents.get(payload.expected.id) === payload.expected
4118 ? captureFinalEvents(payload.events).pipe(
4119 Effect.as({ committed: true, storedEvents: [] }),
4120 )
4121 : Effect.succeed({ committed: false, storedEvents: [] }),
4122 writeIfTurnItemUnchanged: (payload) =>
4123 stored.turnItems.get(payload.expected.id) === payload.expected
4124 ? captureFinalEvents(payload.events).pipe(
4125 Effect.as({ committed: true, storedEvents: [] }),
4126 )
4127 : Effect.succeed({ committed: false, storedEvents: [] }),
4128 }),
4129 IdAllocator.layer,
4130 ThreadCommandExecutor.layer,
4131 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
4132 ingestNormalized: (request) =>
4133 store(request.event).pipe(
4134 Effect.andThen(input.ingestNormalized?.(request) ?? Effect.succeed([])),
4135 ),
4136 }),
4137 ServerSettings.layerTest(),
4138 Layer.succeed(RunFinalizationService.RunFinalizationObserver, {
4139 refresh: () => Effect.void,
4140 refreshAfterTurn: () =>
4141 Ref.update(observed, (current) => [...current, "pull-requests-refreshed"]).pipe(
4142 Effect.andThen(input.refreshAfterTurn ?? Effect.void),
4143 ),
4144 }),
4145 ),
4146 ),
4147 Layer.provideMerge(PreviewAgentActivity.layer),
4148 );
4149
4150 const previewActivityEnded = yield* Effect.gen(function* () {
4151 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
4152 const start = runExecution.startRootRun({
4153 commandId: CommandId.make(`command:${input.key}`),
4154 appThread: rootAppThread(ids.threadId),
4155 providerSessionId: ProviderSessionId.make(`session:${input.key}`),
4156 session: {
4157 driver,
4158 events: Stream.empty,
4159 subscribeEvents: Effect.succeed({
4160 events:
4161 input.events?.(ids) ??
4162 Stream.fromIterable([
4163 ...(input.seedOpenSubagent
4164 ? [
4165 { type: "subagent.updated", driver, subagent: runningSubagent } as const,
4166 {
4167 type: "node.updated",
4168 driver,
4169 node: makeRunOwnedSubagentNodeFixture({ ids, status: "running" }),
4170 } as const,
4171 {
4172 type: "turn_item.updated",
4173 driver,
4174 turnItem: makeRunOwnedSubagentTurnItemFixture({
4175 ids,
4176 providerInstanceId,
4177 childThreadId: ids.childThreadId,
4178 driver,
4179 status: "running",
4180 }),
4181 } as const,
4182 ]
4183 : []),
4184 rootTerminalEvent(ids, "interrupted"),
4185 ] satisfies ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>),
4186 close: Deferred.succeed(ingestionDone, undefined).pipe(
4187 Effect.andThen(input.onSubscriptionClose ?? Effect.void),
4188 ),
4189 }),
4190 startTurn: input.startTurn ?? (() => Effect.void),
4191 interruptTurn: input.interruptTurn ?? (() => Effect.void),
4192 hasPendingBackgroundWorkForThread: input.hasPendingBackgroundWorkForThread,
4193 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
4194 run: {
4195 id: ids.runId,
4196 threadId: ids.threadId,
4197 ordinal: 1,
4198 providerInstanceId,
4199 } as OrchestrationV2Run,
4200 rootNode: {
4201 id: ids.rootNodeId,
4202 providerTurnId: ids.rootProviderTurnId,
4203 } as OrchestrationV2ExecutionNode,
4204 checkpointScope: {
4205 id: CheckpointScopeId.make(`checkpoint-scope:${input.key}`),
4206 } as OrchestrationV2CheckpointScope,
4207 providerThread: {
4208 id: ids.providerThreadId,
4209 driver,
4210 } as OrchestrationV2ProviderThread,
4211 attempt: {
4212 id: ids.attemptId,
4213 providerTurnId:
4214 input.initialProviderTurnId === undefined
4215 ? ids.rootProviderTurnId
4216 : input.initialProviderTurnId,
4217 } as OrchestrationV2RunAttempt,
4218 attemptId: ids.attemptId,
4219 providerTurnOrdinal: 1,
4220 shouldFinalizeRun: input.shouldFinalizeRun,
4221 loadUnsettledWork,
4222 ...(input.loadRunInterruptReason === undefined
4223 ? {}
4224 : { loadRunInterruptReason: input.loadRunInterruptReason }),
4225 ...(input.shouldStartProviderTurn === undefined
4226 ? {}
4227 : { shouldStartProviderTurn: input.shouldStartProviderTurn }),
4228 ...(input.hasUnpairedRunInterruptRequest === undefined
4229 ? {}
4230 : {
4231 hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest,
4232 }),
4233 ...(input.hasWorkspaceMoved === undefined
4234 ? {}
4235 : { hasWorkspaceMoved: input.hasWorkspaceMoved }),
4236 message: {
4237 messageId: MessageId.make(`message:${input.key}`),
4238 text: input.messageText ?? "interrupt projection",
4239 attachments: [],
4240 createdBy: "user",
4241 creationSource: "web",
4242 },
4243 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
4244 runtimePolicy: {
4245 runtimeMode: "full-access",
4246 interactionMode: "default",
4247 cwd: process.cwd(),
4248 approvalPolicy: "never",
4249 sandboxPolicy: {
4250 type: "readOnly",
4251 access: { type: "fullAccess" },
4252 networkAccess: false,
4253 },
4254 },
4255 });
4256 yield* start.pipe(
4257 Effect.catch((error) => input.onStartFailure?.(error) ?? Effect.fail(error)),
4258 );
4259 yield* input.afterStart?.(ingestionDone) ?? Effect.void;
4260 yield* Deferred.await(ingestionDone);
4261 const activity = yield* PreviewAgentActivity.PreviewAgentActivity;
4262 return !(yield* activity.active(ids.threadId));
4263 }).pipe(Effect.provide(layerTest));
4264
4265 return {
4266 previewActivityEnded,
4267 written: yield* Ref.get(writtenItems),
4268 finalEvents: yield* Ref.get(finalEvents),
4269 observed: yield* Ref.get(observed),
4270 submittedEffects: yield* Ref.get(submittedEffects),
4271 committedEffects: yield* Ref.get(committedEffects),
4272 };
4273 });
4274 }
4275
4276 interface BackgroundScenarioIds {
4277 readonly threadId: ThreadId;
4278 readonly childThreadId: ThreadId;
4279 readonly runId: RunId;
4280 readonly attemptId: RunAttemptId;
4281 readonly providerThreadId: ProviderThreadId;
4282 readonly rootProviderTurnId: ProviderTurnId;
4283 readonly rootNodeId: NodeId;
4284 readonly itemId: TurnItemId;
4285 readonly childItemId: TurnItemId;
4286 readonly subagentNodeId: NodeId;
4287 }
4288
4289 function backgroundScenarioIds(key: string): BackgroundScenarioIds {
4290 return {
4291 threadId: ThreadId.make(`thread:${key}`),
4292 childThreadId: ThreadId.make(`thread:${key}:child`),
4293 runId: RunId.make(`run:${key}`),
4294 attemptId: RunAttemptId.make(`attempt:${key}`),
4295 providerThreadId: ProviderThreadId.make(`provider-thread:${key}`),
4296 rootProviderTurnId: ProviderTurnId.make(`provider-turn:${key}`),
4297 rootNodeId: NodeId.make(`node:${key}`),
4298 itemId: TurnItemId.make(`turn-item:${key}`),
4299 childItemId: TurnItemId.make(`turn-item:${key}:child`),
4300 subagentNodeId: NodeId.make(`node:${key}:subagent`),
4301 };
4302 }
4303
4304 function childThreadCreatedEvent(
4305 ids: BackgroundScenarioIds,
4306 ): ProviderAdapter.ProviderAdapterV2Event {
4307 return {
4308 type: "app_thread.created",
4309 driver,
4310 appThread: {
4311 id: ids.childThreadId,
4312 lineage: {
4313 parentThreadId: ids.threadId,
4314 relationshipToParent: "subagent",
4315 rootThreadId: ids.threadId,
4316 },
4317 },
4318 } as ProviderAdapter.ProviderAdapterV2Event;
4319 }
4320
4321 function childBackgroundTurnItemEvent(
4322 ids: BackgroundScenarioIds,
4323 status: "running" | "completed" | "idle",
4324 ordinal: number,
4325 ): ProviderAdapter.ProviderAdapterV2Event {
4326 return {
4327 type: "turn_item.updated",
4328 driver,
4329 turnItem: {
4330 id: ids.childItemId,
4331 threadId: ids.childThreadId,
4332 runId: null,
4333 providerTurnId: null,
4334 ordinal,
4335 type: "command_execution",
4336 status,
4337 },
4338 } as ProviderAdapter.ProviderAdapterV2Event;
4339 }
4340
4341 function backgroundTurnItemEvent(
4342 ids: BackgroundScenarioIds,
4343 type: "command_execution" | "dynamic_tool" | "subagent",
4344 status: "running" | "completed" | "idle",
4345 ordinal: number,
4346 itemId?: TurnItemId,
4347 ): ProviderAdapter.ProviderAdapterV2Event {
4348 return {
4349 type: "turn_item.updated",
4350 driver,
4351 turnItem: {
4352 id: itemId ?? ids.itemId,
4353 threadId: ids.threadId,
4354 runId: ids.runId,
4355 providerTurnId: ids.rootProviderTurnId,
4356 ordinal,
4357 type,
4358 status,
4359 },
4360 } as ProviderAdapter.ProviderAdapterV2Event;
4361 }
4362
4363 function backgroundTurnItemEventForRun(
4364 ids: BackgroundScenarioIds,
4365 runId: RunId,
4366 type: "command_execution" | "dynamic_tool" | "subagent",
4367 status: "running" | "completed" | "idle",
4368 ordinal: number,
4369 ): ProviderAdapter.ProviderAdapterV2Event {
4370 const event = backgroundTurnItemEvent(ids, type, status, ordinal);
4371 if (event.type !== "turn_item.updated") {
4372 return event;
4373 }
4374 return { ...event, turnItem: { ...event.turnItem, runId } };
4375 }
4376
4377 function subagentEvent(
4378 ids: BackgroundScenarioIds,
4379 status: "running" | "completed" | "idle",
4380 ): ProviderAdapter.ProviderAdapterV2Event {
4381 return {
4382 type: "subagent.updated",
4383 driver,
4384 subagent: {
4385 id: ids.subagentNodeId,
4386 threadId: ids.threadId,
4387 runId: ids.runId,
4388 status,
4389 },
4390 } as ProviderAdapter.ProviderAdapterV2Event;
4391 }
4392
4393 function makeRunOwnedSubagentFixture(input: {
4394 readonly ids: BackgroundScenarioIds;
4395 readonly providerInstanceId: ProviderInstanceId;
4396 readonly childThreadId: ThreadId;
4397 readonly driver: typeof driver;
4398 readonly status: "running" | "interrupted";
4399 }): OrchestrationV2Subagent {
4400 const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z");
4401 return {
4402 id: input.ids.subagentNodeId,
4403 threadId: input.ids.threadId,
4404 runId: input.ids.runId,
4405 parentNodeId: input.ids.rootNodeId,
4406 origin: "provider_native",
4407 createdBy: "agent",
4408 driver: input.driver,
4409 providerInstanceId: input.providerInstanceId,
4410 providerThreadId: null,
4411 childThreadId: input.childThreadId,
4412 nativeTaskRef: {
4413 driver: input.driver,
4414 nativeId: `task:${input.ids.subagentNodeId}`,
4415 strength: "strong",
4416 },
4417 prompt: "hold",
4418 title: "Live-test subagent hold",
4419 model: null,
4420 status: input.status,
4421 result: null,
4422 startedAt: now,
4423 completedAt: null,
4424 updatedAt: now,
4425 };
4426 }
4427
4428 function makeRunOwnedSubagentTurnItemFixture(input: {
4429 readonly ids: BackgroundScenarioIds;
4430 readonly providerInstanceId: ProviderInstanceId;
4431 readonly childThreadId: ThreadId;
4432 readonly driver: typeof driver;
4433 readonly status: "running" | "interrupted";
4434 }): Extract<OrchestrationV2TurnItem, { type: "subagent" }> {
4435 const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z");
4436 return {
4437 id: input.ids.itemId,
4438 threadId: input.ids.threadId,
4439 runId: input.ids.runId,
4440 nodeId: input.ids.subagentNodeId,
4441 providerThreadId: input.ids.providerThreadId,
4442 providerTurnId: input.ids.rootProviderTurnId,
4443 nativeItemRef: {
4444 driver: input.driver,
4445 nativeId: `task:${input.ids.subagentNodeId}`,
4446 strength: "strong",
4447 },
4448 parentItemId: null,
4449 ordinal: 3,
4450 status: input.status,
4451 title: "Live-test subagent hold",
4452 startedAt: now,
4453 completedAt: null,
4454 updatedAt: now,
4455 type: "subagent",
4456 subagentId: input.ids.subagentNodeId,
4457 origin: "provider_native",
4458 driver: input.driver,
4459 providerInstanceId: input.providerInstanceId,
4460 childThreadId: input.childThreadId,
4461 prompt: "hold",
4462 result: null,
4463 };
4464 }
4465
4466 function makeLinkedChildTurnItemFixture(input: {
4467 readonly ids: BackgroundScenarioIds;
4468 readonly driver: typeof driver;
4469 readonly type: "assistant_message" | "command_execution" | "reasoning";
4470 }): OrchestrationV2TurnItem {
4471 const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z");
4472 const base = {
4473 id: input.ids.childItemId,
4474 threadId: input.ids.childThreadId,
4475 runId: null,
4476 nodeId: NodeId.make(`${input.ids.subagentNodeId}:child-root`),
4477 providerThreadId: null,
4478 providerTurnId: null,
4479 nativeItemRef: null,
4480 parentItemId: null,
4481 ordinal: 1,
4482 status: "running" as const,
4483 title: "Working",
4484 startedAt: now,
4485 completedAt: null,
4486 updatedAt: now,
4487 };
4488 if (input.type === "assistant_message") {
4489 return {
4490 ...base,
4491 type: "assistant_message",
4492 messageId: MessageId.make(`${input.ids.childItemId}:message`),
4493 text: "partial response",
4494 streaming: true,
4495 };
4496 }
4497 if (input.type === "reasoning") {
4498 return {
4499 ...base,
4500 type: "reasoning",
4501 text: "partial progress",
4502 streaming: true,
4503 };
4504 }
4505 return {
4506 ...base,
4507 type: "command_execution",
4508 input: "sleep 300",
4509 };
4510 }
4511
4512 function makeRunOwnedSubagentNodeFixture(input: {
4513 readonly ids: BackgroundScenarioIds;
4514 readonly status: "running" | "interrupted";
4515 }): OrchestrationV2ExecutionNode {
4516 const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z");
4517 return {
4518 id: input.ids.subagentNodeId,
4519 threadId: input.ids.threadId,
4520 runId: input.ids.runId,
4521 parentNodeId: input.ids.rootNodeId,
4522 rootNodeId: input.ids.rootNodeId,
4523 kind: "subagent",
4524 status: input.status,
4525 countsForRun: false,
4526 providerThreadId: input.ids.providerThreadId,
4527 providerTurnId: input.ids.rootProviderTurnId,
4528 nativeItemRef: {
4529 driver,
4530 nativeId: `task:${input.ids.subagentNodeId}`,
4531 strength: "strong",
4532 },
4533 runtimeRequestId: null,
4534 checkpointScopeId: null,
4535 startedAt: now,
4536 completedAt: null,
4537 };
4538 }
4539
4540 function makeRunOwnedSubagentChildNodeFixture(input: {
4541 readonly ids: BackgroundScenarioIds;
4542 readonly status: "running" | "interrupted";
4543 }): OrchestrationV2ExecutionNode {
4544 const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z");
4545 return {
4546 id: NodeId.make(`${input.ids.subagentNodeId}:child-root`),
4547 threadId: input.ids.childThreadId,
4548 runId: null,
4549 parentNodeId: null,
4550 rootNodeId: NodeId.make(`${input.ids.subagentNodeId}:child-root`),
4551 kind: "root_turn",
4552 status: input.status,
4553 countsForRun: false,
4554 providerThreadId: null,
4555 providerTurnId: null,
4556 nativeItemRef: null,
4557 runtimeRequestId: null,
4558 checkpointScopeId: null,
4559 startedAt: now,
4560 completedAt: null,
4561 };
4562 }
4563
4564 function rootTerminalEvent(
4565 ids: BackgroundScenarioIds,
4566 status: "completed" | "interrupted" | "cancelled" | "failed",
4567 ): ProviderAdapter.ProviderAdapterV2Event {
4568 const common = {
4569 type: "turn.terminal" as const,
4570 driver,
4571 providerThreadId: ids.providerThreadId,
4572 providerTurnId: ids.rootProviderTurnId,
4573 runOrdinal: 1,
4574 threadDisposition: "reusable" as const,
4575 };
4576 return status === "failed"
4577 ? {
4578 ...common,
4579 status,
4580 failureItemOrdinal: 101,
4581 failure: {
4582 class: "provider_error",
4583 message: "Provider failed",
4584 code: null,
4585 retryable: null,
4586 },
4587 }
4588 : { ...common, status, failure: null };
4589 }
4590
4591 function runBackgroundItemScenario(
4592 key: string,
4593 makeEvents: (
4594 ids: BackgroundScenarioIds,
4595 ) =>
4596 | ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>
4597 | Stream.Stream<ProviderAdapter.ProviderAdapterV2Event>,
4598 options?: {
4599 readonly onIngest?: (
4600 event: ProviderAdapter.ProviderAdapterV2Event,
4601 write: { readonly inSharedCommit: boolean },
4602 ) => Effect.Effect<void>;
4603 // Consumes through the real commit coordinator, as the event sink does.
4604 readonly batched?: boolean;
4605 readonly keepEventStreamOpen?: boolean;
4606 readonly hasPendingBackgroundWorkForThread?: (
4607 providerThread: OrchestrationV2ProviderThread,
4608 ) => Effect.Effect<boolean>;
4609 readonly loadInheritedBackgroundTurnItems?: () => Effect.Effect<
4610 ReadonlyArray<{ readonly id: TurnItemId; readonly runId: RunId }>
4611 >;
4612 readonly onSubscribe?: Effect.Effect<void>;
4613 readonly onEventsConsumed?: Effect.Effect<void>;
4614 // Owner-gated root snapshots store nothing, as after a newer run claims the thread.
4615 readonly providerThreadOwnerLost?: boolean;
4616 readonly afterEvents?: (
4617 activity: PreviewAgentActivity.PreviewAgentActivity["Service"],
4618 ) => Effect.Effect<void>;
4619 readonly onFinalized?: (
4620 events: ReadonlyArray<OrchestrationV2DomainEvent>,
4621 ) => Effect.Effect<void>;
4622 },
4623 ) {
4624 return Effect.gen(function* () {
4625 const ids = backgroundScenarioIds(key);
4626 const providerInstanceId = ProviderInstanceId.make("codex");
4627 const observed = yield* Ref.make<ReadonlyArray<string>>([]);
4628 const ingestionDone = yield* Deferred.make<void>();
4629 const previewActivity = yield* PreviewAgentActivity.PreviewAgentActivity.pipe(
4630 Effect.provide(PreviewAgentActivity.layer),
4631 );
4632 let inSharedCommit = false;
4633 const coordinator =
4634 options?.batched === true
4635 ? yield* ProviderCommitBatch.make(
4636 (work) =>
4637 Effect.suspend(() => {
4638 inSharedCommit = true;
4639 return work;
4640 }).pipe(
4641 Effect.ensuring(
4642 Effect.sync(() => {
4643 inSharedCommit = false;
4644 }),
4645 ),
4646 ),
4647 true,
4648 )
4649 : undefined;
4650 const layerTest = RunExecutionService.layer.pipe(
4651 Layer.provide(
4652 Layer.mergeAll(
4653 McpAppModelContext.layerEmpty,
4654 Layer.succeed(PreviewAgentActivity.PreviewAgentActivity, previewActivity),
4655 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
4656 layerTurnItemPositionsTest,
4657 mockEventSink({
4658 ...(coordinator === undefined ? {} : { consumeProviderEvents: coordinator.consume }),
4659 write: () => Effect.succeed([]),
4660 writeWithEffects: (input) =>
4661 Effect.gen(function* () {
4662 if (
4663 input.events.some(
4664 (event) => event.type === "run.updated" && event.runId === ids.runId,
4665 )
4666 ) {
4667 yield* Ref.update(observed, (current) => [...current, "root-finalized"]);
4668 yield* options?.onFinalized?.(input.events) ?? Effect.void;
4669 }
4670 return [];
4671 }),
4672 writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }),
4673 }),
4674 IdAllocator.layer,
4675 ThreadCommandExecutor.layer,
4676 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
4677 ingestNormalized: (input) => {
4678 const write = Effect.gen(function* () {
4679 const event = input.event;
4680 yield* options?.onIngest?.(event, { inSharedCommit }) ?? Effect.void;
4681 if (event.type === "turn_item.updated") {
4682 yield* Ref.update(observed, (current) => [
4683 ...current,
4684 `turn_item:${event.turnItem.status}`,
4685 ]);
4686 }
4687 if (event.type === "subagent.updated") {
4688 yield* Ref.update(observed, (current) => [
4689 ...current,
4690 `subagent:${event.subagent.status}`,
4691 ]);
4692 }
4693 // Only the count of stored events matters to the run.
4694 return event.type === "provider_thread.updated" &&
4695 !(
4696 options?.providerThreadOwnerLost === true &&
4697 input.writeIfProviderThreadOwner !== undefined
4698 )
4699 ? [{} as OrchestrationV2StoredEvent]
4700 : [];
4701 });
4702 // A write outside a batch waits for queued snapshots, as the event sink's does.
4703 return coordinator === undefined ? write : coordinator.barrier(write);
4704 },
4705 }),
4706 ServerSettings.layerTest(),
4707 ),
4708 ),
4709 );
4710
4711 yield* Effect.gen(function* () {
4712 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
4713 yield* runExecution.startRootRun({
4714 commandId: CommandId.make(`command:${key}`),
4715 appThread: rootAppThread(ids.threadId),
4716 providerSessionId: ProviderSessionId.make(`session:${key}`),
4717 session: {
4718 events: Stream.empty,
4719 ...(options?.hasPendingBackgroundWorkForThread === undefined
4720 ? {}
4721 : { hasPendingBackgroundWorkForThread: options.hasPendingBackgroundWorkForThread }),
4722 subscribeEvents: Effect.gen(function* () {
4723 yield* options?.onSubscribe ?? Effect.void;
4724 const source = makeEvents(ids);
4725 const events = (Stream.isStream(source) ? source : Stream.fromIterable(source)).pipe(
4726 Stream.concat(
4727 Stream.fromEffect(
4728 Effect.andThen(
4729 options?.onEventsConsumed ?? Effect.void,
4730 options?.afterEvents?.(previewActivity) ?? Effect.void,
4731 ),
4732 ).pipe(Stream.drain),
4733 ),
4734 );
4735 return {
4736 events:
4737 options?.keepEventStreamOpen === true
4738 ? events.pipe(Stream.concat(Stream.never))
4739 : events,
4740 close: Deferred.succeed(ingestionDone, undefined),
4741 };
4742 }),
4743 startTurn: () => Effect.void,
4744 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
4745 run: {
4746 id: ids.runId,
4747 threadId: ids.threadId,
4748 ordinal: 1,
4749 providerInstanceId,
4750 } as OrchestrationV2Run,
4751 rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode,
4752 checkpointScope: {
4753 id: CheckpointScopeId.make(`checkpoint-scope:${key}`),
4754 } as OrchestrationV2CheckpointScope,
4755 providerThread: {
4756 id: ids.providerThreadId,
4757 driver,
4758 } as OrchestrationV2ProviderThread,
4759 attempt: {
4760 id: ids.attemptId,
4761 providerTurnId: ids.rootProviderTurnId,
4762 } as OrchestrationV2RunAttempt,
4763 attemptId: ids.attemptId,
4764 ...(options?.loadInheritedBackgroundTurnItems === undefined
4765 ? {}
4766 : {
4767 loadInheritedBackgroundTurnItems: options.loadInheritedBackgroundTurnItems,
4768 }),
4769 providerTurnOrdinal: 1,
4770 message: {
4771 messageId: MessageId.make(`message:${key}:user`),
4772 text: "Start a background item and finish.",
4773 attachments: [],
4774 createdBy: "user",
4775 creationSource: "web",
4776 },
4777 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
4778 runtimePolicy: {
4779 runtimeMode: "full-access",
4780 interactionMode: "default",
4781 cwd: process.cwd(),
4782 approvalPolicy: "never",
4783 sandboxPolicy: {
4784 type: "readOnly",
4785 access: { type: "fullAccess" },
4786 networkAccess: false,
4787 },
4788 },
4789 });
4790 }).pipe(Effect.provide(layerTest));
4791
4792 // No timeout: tests advance the TestClock, which would fire it early.
4793 yield* Deferred.await(ingestionDone);
4794
4795 return yield* Ref.get(observed);
4796 });
4797 }
4798
4799 it.effect("keeps a failed root's still-running native children live while the run settles", () =>
4800 Effect.gen(function* () {
4801 const finalized = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);
4802 const childTurn = (
4803 ids: BackgroundScenarioIds,
4804 status: "running" | "completed",
4805 ): ProviderAdapter.ProviderAdapterV2Event =>
4806 ({
4807 type: "provider_turn.updated",
4808 driver,
4809 threadId: ids.childThreadId,
4810 providerTurn: {
4811 id: ProviderTurnId.make(`${ids.rootProviderTurnId}:child`),
4812 providerThreadId: ProviderThreadId.make(`${ids.providerThreadId}:child`),
4813 nodeId: NodeId.make(`${ids.subagentNodeId}:child-root`),
4814 runAttemptId: null,
4815 status,
4816 },
4817 }) as ProviderAdapter.ProviderAdapterV2Event;
4818 const observed = yield* runBackgroundItemScenario(
4819 "failed-root-live-child",
4820 (ids) => [
4821 childThreadCreatedEvent(ids),
4822 subagentEvent(ids, "running"),
4823 childTurn(ids, "running"),
4824 { ...rootTerminalEvent(ids, "failed"), childTurnsContinue: true },
4825 subagentEvent(ids, "completed"),
4826 childTurn(ids, "completed"),
4827 ],
4828 {
4829 keepEventStreamOpen: true,
4830 onFinalized: (events) => Ref.set(finalized, events),
4831 },
4832 );
4833
4834 assert.deepEqual(observed, ["subagent:running", "root-finalized", "subagent:completed"]);
4835 const written = yield* Ref.get(finalized);
4836 assert.deepEqual(
4837 written.flatMap((event) => (event.type === "run.updated" ? [event.payload.status] : [])),
4838 ["failed"],
4839 );
4840 assert.isFalse(
4841 written.some((event) => event.type === "subagent.updated"),
4842 "a child that outlives the failed root must not be settled with it",
4843 );
4844 }),
4845 );
4846
4847 it.effect.each([
4848 { name: "on time", wakesEarly: false },
4849 // Timers run on the monotonic clock, so one can fire while the wall clock still reads a
4850 // millisecond before its deadline.
4851 { name: "before the wall clock reaches the deadline", wakesEarly: true },
4852 ])("delivers held text while an open-prompt provider idles, waking $name", ({ wakesEarly }) =>
4853 Effect.gen(function* () {
4854 const key = `idle-paragraph-${wakesEarly}`;
4855 const ids = backgroundScenarioIds(key);
4856 const now = yield* DateTime.now;
4857 const providerEvents = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
4858 const delivered = yield* Ref.make<ReadonlyArray<string>>([]);
4859 const paragraphDelivered = yield* Deferred.make<void>();
4860 const nextParagraphDelivered = yield* Deferred.make<void>();
4861 const clock = yield* Clock.Clock;
4862 const assistantText = (text: string, streaming: boolean) =>
4863 [
4864 {
4865 type: "message.updated",
4866 driver,
4867 message: {
4868 createdBy: "agent",
4869 creationSource: "provider",
4870 id: MessageId.make(`message:${key}`),
4871 threadId: ids.threadId,
4872 runId: ids.runId,
4873 nodeId: ids.rootNodeId,
4874 role: "assistant",
4875 text,
4876 attachments: [],
4877 streaming,
4878 createdAt: now,
4879 updatedAt: now,
4880 },
4881 },
4882 {
4883 type: "turn_item.updated",
4884 driver,
4885 turnItem: {
4886 ...makeLinkedChildTurnItemFixture({ ids, driver, type: "assistant_message" }),
4887 id: ids.itemId,
4888 threadId: ids.threadId,
4889 runId: ids.runId,
4890 nodeId: ids.rootNodeId,
4891 providerTurnId: ids.rootProviderTurnId,
4892 text,
4893 streaming,
4894 },
4895 },
4896 ] as ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>;
4897
4898 const blockerStarted = yield* Deferred.make<void>();
4899 const releaseBlocker = yield* Deferred.make<void>();
4900 const blocker = streamedMessageEvent({
4901 id: `message:${key}:blocker`,
4902 threadId: ids.threadId,
4903 runId: ids.runId,
4904 role: "assistant",
4905 text: "Other",
4906 streaming: false,
4907 });
4908 const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(providerEvents), {
4909 batched: true,
4910 onIngest: (event, write) =>
4911 event.type === "message.updated" && event.message.text === "Other"
4912 ? Deferred.succeed(blockerStarted, undefined).pipe(
4913 Effect.andThen(Deferred.await(releaseBlocker)),
4914 )
4915 : event.type === "message.updated" || event.type === "turn_item.updated"
4916 ? Effect.gen(function* () {
4917 const message = event.type === "message.updated" ? event.message : event.turnItem;
4918 const text = "text" in message ? message.text : "";
4919 const streaming = "streaming" in message && message.streaming;
4920 yield* Ref.update(delivered, (current) => [
4921 ...current,
4922 `${event.type}:${streaming ? "streaming" : "final"}:${JSON.stringify(text)}${
4923 write.inSharedCommit ? " in commit" : ""
4924 }`,
4925 ]);
4926 if (event.type === "turn_item.updated") {
4927 yield* Deferred.succeed(paragraphDelivered, undefined);
4928 if (text === "First.\n\nWaiting on the build.\n\n")
4929 yield* Deferred.succeed(nextParagraphDelivered, undefined);
4930 }
4931 })
4932 : Effect.void,
4933 }).pipe(
4934 Effect.provideService(Clock.Clock, {
4935 ...clock,
4936 sleep: (duration) =>
4937 clock.sleep(wakesEarly ? Duration.subtract(duration, Duration.millis(1)) : duration),
4938 }),
4939 Effect.forkChild,
4940 );
4941 // Waking a sleep runs ingestion until it suspends, so an idle write has landed by now.
4942 const lastShown = Ref.get(delivered).pipe(Effect.map((shown) => shown.at(-1)));
4943
4944 yield* Queue.offerAll(providerEvents, assistantText("First.\n\nWaiting on the build", true));
4945 yield* Deferred.await(paragraphDelivered);
4946 // The provider keeps its prompt open while a backgrounded command runs.
4947 yield* TestClock.adjust("2 seconds");
4948 assert.strictEqual(
4949 yield* lastShown,
4950 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build"',
4951 );
4952 // The next paragraph's tail still flushes on a later pause.
4953 yield* Queue.offerAll(
4954 providerEvents,
4955 assistantText("First.\n\nWaiting on the build.\n\nRunning", true),
4956 );
4957 yield* Deferred.await(nextParagraphDelivered);
4958 yield* TestClock.adjust("1 second");
4959 assert.strictEqual(
4960 yield* lastShown,
4961 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\nRunning"',
4962 );
4963 yield* Queue.offerAll(providerEvents, [
4964 ...assistantText("First.\n\nWaiting on the build.\n\nRunning tests", true),
4965 blocker,
4966 ...assistantText("First.\n\nWaiting on the build.\n\nRunning tests passed.", false),
4967 ]);
4968 yield* Deferred.await(blockerStarted);
4969 // The appended text comes due while its final snapshots wait behind the blocked write.
4970 yield* TestClock.adjust("1 second");
4971 yield* Deferred.succeed(releaseBlocker, undefined);
4972 yield* Queue.offer(providerEvents, rootTerminalEvent(ids, "completed"));
4973 yield* Fiber.join(scenario);
4974
4975 // Snapshots share the batch commit; the idle write commits alone, before it counts as shown.
4976 // The final snapshots win over the stale append that came due behind them.
4977 assert.deepEqual(yield* Ref.get(delivered), [
4978 `message.updated:streaming:"First.\\n\\n" in commit`,
4979 `turn_item.updated:streaming:"First.\\n\\n" in commit`,
4980 'message.updated:streaming:"First.\\n\\nWaiting on the build"',
4981 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build"',
4982 `message.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\n" in commit`,
4983 `turn_item.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\n" in commit`,
4984 'message.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\nRunning"',
4985 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\nRunning"',
4986 `message.updated:final:"First.\\n\\nWaiting on the build.\\n\\nRunning tests passed." in commit`,
4987 `turn_item.updated:final:"First.\\n\\nWaiting on the build.\\n\\nRunning tests passed." in commit`,
4988 ]);
4989 }),
4990 );
4991
4992 const heldChildText = "partial response";
4993
4994 function childAssistantSnapshot(
4995 ids: BackgroundScenarioIds,
4996 ): ProviderAdapter.ProviderAdapterV2Event {
4997 return {
4998 type: "turn_item.updated",
4999 driver,
5000 turnItem: {
Showing first 5,000 of 5,237 lines. View raw