The best way to control Claude Code, Codex and more from any computer or phone

main
ts 1,147 lines 41.4 KB
Raw
1 import { assert, it } from "@effect/vitest";
2 import {
3 CommandId,
4 EventId,
5 MessageId,
6 NodeId,
7 ProjectId,
8 ProviderDriverKind,
9 ProviderInstanceId,
10 ProviderThreadId,
11 ProviderTurnId,
12 RunAttemptId,
13 RunId,
14 RuntimeRequestId,
15 ThreadId,
16 TurnItemId,
17 type OrchestrationV2DomainEvent,
18 } from "@t3tools/contracts";
19 import * as DateTime from "effect/DateTime";
20 import * as Effect from "effect/Effect";
21 import * as Fiber from "effect/Fiber";
22 import * as Queue from "effect/Queue";
23 import * as Stream from "effect/Stream";
24 import * as TestClock from "effect/testing/TestClock";
25 import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
26 import * as EffectWorker from "./EffectWorker.ts";
27 import * as EventSink from "./EventSink.ts";
28 import * as Orchestrator from "./Orchestrator.ts";
29 import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts";
30 import * as ProviderSessionManager from "./ProviderSessionManager.ts";
31 import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts";
32 import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace";
33 import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
34
35 const driver = ProviderDriverKind.make("codex");
36 const instanceId = ProviderInstanceId.make("codex");
37 const modelSelection = { instanceId, model: "test-model" };
38
39 const stopEarlierBackgroundWork = ({
40 wakeAfterStop = false,
41 failedStart = false,
42 stopWithQueue,
43 olderStart = false,
44 backgroundWorkAfterStop,
45 stalledRun,
46 workflowMember = false,
47 }: {
48 readonly wakeAfterStop?: boolean;
49 readonly failedStart?: boolean;
50 readonly backgroundWorkAfterStop?: "running" | "unknown";
51 readonly stopWithQueue?: "thread.stop" | "run.interrupt";
52 readonly olderStart?: boolean;
53 readonly workflowMember?: boolean;
54 readonly stalledRun?: "missing-session-terminal" | "returned-interrupt" | "superseded-attempt";
55 }) =>
56 Effect.gen(function* () {
57 const cwd = yield* checkpointWorkspace("background-work-stop");
58 const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
59 const started: ProviderAdapter.ProviderAdapterV2TurnInput[] = [];
60 const interrupts: ProviderAdapter.ProviderAdapterV2InterruptInput[] = [];
61 const interruptCalls = yield* Queue.unbounded<void>();
62 const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = {
63 instanceId,
64 driver,
65 getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2),
66 planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }),
67 openSession: (input) =>
68 Effect.gen(function* () {
69 const now = yield* DateTime.now;
70 return {
71 instanceId,
72 driver,
73 providerSessionId: input.providerSessionId,
74 providerSession: {
75 id: input.providerSessionId,
76 driver,
77 providerInstanceId: instanceId,
78 status: "ready",
79 cwd,
80 model: modelSelection.model,
81 capabilities: CodexProviderCapabilitiesV2,
82 createdAt: now,
83 updatedAt: now,
84 lastError: null,
85 },
86 events: Stream.fromQueue(events),
87 ...(failedStart
88 ? {
89 hasPendingBackgroundWorkForThread: (providerThread) =>
90 Effect.gen(function* () {
91 if (
92 !interrupts.some((input) => input.providerThread.id === providerThread.id)
93 ) {
94 return false;
95 }
96 if (backgroundWorkAfterStop === "unknown") {
97 return yield* Effect.die("Background work probe failed");
98 }
99 return backgroundWorkAfterStop === "running";
100 }),
101 }
102 : {}),
103 ensureThread: ({ threadId }) =>
104 Effect.succeed({
105 id: ProviderThreadId.make(`provider-thread:codex:${threadId}`),
106 driver,
107 providerInstanceId: instanceId,
108 providerSessionId: input.providerSessionId,
109 appThreadId: threadId,
110 ownerNodeId: null,
111 nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" },
112 nativeConversationHeadRef: null,
113 status: "idle",
114 firstRunOrdinal: null,
115 lastRunOrdinal: null,
116 handoffIds: [],
117 forkedFrom: null,
118 createdAt: now,
119 updatedAt: now,
120 }),
121 resumeThread: ({ providerThread }) => Effect.succeed(providerThread),
122 startTurn: (turn) =>
123 Effect.gen(function* () {
124 started.push(turn);
125 yield* Queue.offer(events, {
126 type: "provider_turn.updated",
127 driver,
128 providerTurn: {
129 id: ProviderTurnId.make(`provider-turn:${turn.attemptId}`),
130 providerThreadId: turn.providerThread.id,
131 nodeId: turn.rootNodeId,
132 runAttemptId: turn.attemptId,
133 nativeTurnRef: {
134 driver,
135 nativeId: `native:${turn.attemptId}`,
136 strength: "strong",
137 },
138 ordinal: turn.providerTurnOrdinal,
139 status: "running",
140 startedAt: now,
141 completedAt: null,
142 },
143 });
144 }),
145 steerTurn: () => Effect.die("unused"),
146 interruptTurn: (interrupt) =>
147 Effect.sync(() => {
148 interrupts.push(interrupt);
149 }).pipe(Effect.andThen(Queue.offer(interruptCalls, undefined))),
150 respondToRuntimeRequest: () => Effect.die("unused"),
151 readThreadSnapshot: () => Effect.die("unused"),
152 rollbackThread: () => Effect.die("unused"),
153 forkThread: () => Effect.die("unused"),
154 };
155 }),
156 };
157 yield* Effect.gen(function* () {
158 const orchestrator = yield* Orchestrator.OrchestratorV2;
159 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
160 const sink = yield* EventSink.EventSinkV2;
161 const threadId = ThreadId.make("thread:background-work-stop");
162 // This provider never confirms Stop, so Stop gives up waiting at its bound.
163 const drainUnconfirmedStop = Effect.gen(function* () {
164 const drained = yield* Effect.forkChild(worker.drain());
165 yield* Queue.take(interruptCalls);
166 yield* TestClock.adjust("2 seconds");
167 yield* Fiber.join(drained);
168 });
169 const watch = (predicate: (event: OrchestrationV2DomainEvent) => boolean) =>
170 orchestrator.streamDomainEvents.pipe(
171 Stream.filter(predicate),
172 Stream.take(1),
173 Stream.runDrain,
174 Effect.forkScoped,
175 );
176 yield* orchestrator.dispatch({
177 type: "thread.create",
178 commandId: CommandId.make("create"),
179 threadId,
180 projectId: ProjectId.make("project:background-work-stop"),
181 title: "Background work stop",
182 modelSelection,
183 runtimeMode: "full-access",
184 interactionMode: "default",
185 branch: null,
186 worktreePath: cwd,
187 createdBy: "user",
188 creationSource: "web",
189 });
190 const running = yield* watch(
191 (event) => event.type === "provider-turn.updated" && event.payload.status === "running",
192 );
193 yield* orchestrator.dispatch({
194 type: "message.dispatch",
195 commandId: CommandId.make("start-dev-server"),
196 threadId,
197 messageId: MessageId.make("message:start-dev-server"),
198 text: "Start the dev server",
199 attachments: [],
200 dispatchMode: { type: "start_immediately" },
201 createdBy: "user",
202 creationSource: "web",
203 });
204 yield* worker.drain();
205 yield* Fiber.join(running);
206 if (wakeAfterStop) {
207 const send = (id: string, provider = false) =>
208 orchestrator.dispatch({
209 type: "message.dispatch",
210 commandId: CommandId.make(id),
211 threadId,
212 messageId: MessageId.make(`message:${id}`),
213 text: id,
214 attachments: [],
215 dispatchMode: { type: "queue_after_active" },
216 createdBy: provider ? "agent" : "user",
217 creationSource: provider ? "provider" : "web",
218 });
219 yield* send("follow-up");
220 yield* orchestrator.dispatch({
221 type: "thread.stop",
222 commandId: CommandId.make("stop"),
223 threadId,
224 });
225 // Stop is committed, but its provider interrupt has not run yet.
226 yield* send("Background task completed.", true);
227 const [stopped, followUp, wake] = (yield* orchestrator.getThreadProjection(threadId)).runs;
228 assert.equal(stopped?.status, "running");
229 assert.isTrue(followUp?.queueHeld);
230 assert.isTrue(wake?.queueHeld);
231
232 yield* drainUnconfirmedStop;
233 yield* orchestrator.resumeQueuedRuns;
234 yield* worker.drain();
235 const after = yield* orchestrator.getThreadProjection(threadId);
236 assert.deepEqual(
237 after.runs.map((run) => run.status),
238 ["interrupted", "queued", "queued"],
239 );
240 assert.isTrue(after.runs.find((run) => run.id === followUp?.id)?.queueHeld);
241 assert.isTrue(after.runs.find((run) => run.id === wake?.id)?.queueHeld);
242 assert.lengthOf(started, 1);
243 return;
244 }
245 const first = started[0]!;
246 const codexTurn = (yield* orchestrator.getThreadProjection(threadId)).providerTurns[0]!;
247 const devServerId = TurnItemId.make("turn-item:dev-server");
248 const now = yield* DateTime.now;
249 yield* sink.write({
250 events: [
251 {
252 id: EventId.make("dev-server"),
253 type: "turn-item.updated",
254 threadId,
255 runId: first.runId,
256 occurredAt: now,
257 payload: {
258 id: devServerId,
259 threadId,
260 runId: first.runId,
261 nodeId: first.rootNodeId,
262 providerThreadId: codexTurn.providerThreadId,
263 providerTurnId: codexTurn.id,
264 nativeItemRef: null,
265 parentItemId: null,
266 ordinal: 100,
267 status: "running",
268 title: null,
269 startedAt: now,
270 completedAt: null,
271 updatedAt: now,
272 type: "command_execution",
273 input: "vp run dev --share",
274 },
275 },
276 ],
277 });
278 if (stalledRun !== undefined) {
279 const before = yield* orchestrator.getThreadProjection(threadId);
280 const run = before.runs[0]!;
281 const node = before.nodes.find((candidate) => candidate.id === run.rootNodeId)!;
282 const attempt = before.attempts[0]!;
283 if (stalledRun === "missing-session-terminal") {
284 const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
285 const failed = yield* watch(
286 (event) => event.type === "run.updated" && event.payload.status === "failed",
287 );
288 yield* sessions.release({
289 providerSessionId: first.providerThread.providerSessionId!,
290 reason: "runtime_error",
291 });
292 yield* Fiber.join(failed);
293 yield* sink.write({
294 events: [
295 {
296 id: EventId.make("stale-run"),
297 type: "run.updated",
298 threadId,
299 occurredAt: now,
300 payload: run,
301 },
302 {
303 id: EventId.make("stale-node"),
304 type: "node.updated",
305 threadId,
306 occurredAt: now,
307 payload: node,
308 },
309 {
310 id: EventId.make("stale-attempt"),
311 type: "run-attempt.updated",
312 threadId,
313 occurredAt: now,
314 payload: attempt,
315 },
316 ],
317 });
318 }
319 const messageId = MessageId.make("partial-output");
320 const item = before.turnItems.find((candidate) => candidate.id === devServerId)!;
321 assert.ok(item.type === "command_execution");
322 const terminalProviderTurn = stalledRun.endsWith("-terminal");
323 if (terminalProviderTurn) {
324 yield* sink.write({
325 events: [
326 {
327 id: EventId.make("terminal-provider-turn"),
328 type: "provider-turn.updated",
329 threadId,
330 occurredAt: now,
331 payload: { ...codexTurn, status: "completed", completedAt: now },
332 },
333 ...(stalledRun === "missing-session-terminal"
334 ? [
335 {
336 id: EventId.make("completed-dev-server"),
337 type: "turn-item.updated" as const,
338 threadId,
339 occurredAt: now,
340 payload: { ...item, status: "completed" as const, completedAt: now },
341 },
342 ]
343 : []),
344 ],
345 });
346 }
347 yield* sink.write({
348 events: [
349 {
350 id: EventId.make("partial-message"),
351 type: "message.updated",
352 threadId,
353 runId: run.id,
354 occurredAt: now,
355 payload: {
356 id: messageId,
357 threadId,
358 runId: run.id,
359 nodeId: node.id,
360 role: "assistant",
361 text: "Partial output",
362 attachments: [],
363 streaming: true,
364 createdBy: "agent",
365 creationSource: "provider",
366 createdAt: now,
367 updatedAt: now,
368 },
369 },
370 {
371 id: EventId.make("partial-item"),
372 type: "turn-item.updated",
373 threadId,
374 runId: run.id,
375 occurredAt: now,
376 payload: {
377 ...item,
378 providerThreadId: codexTurn.providerThreadId,
379 providerTurnId: codexTurn.id,
380 id: TurnItemId.make("partial-output"),
381 type: "assistant_message",
382 messageId,
383 text: "Partial output",
384 streaming: true,
385 },
386 },
387 ],
388 });
389 // Shaped like CodexAdapterV2's buildApprovalRequestArtifacts output.
390 const approvalRequestId = RuntimeRequestId.make("request:stalled-approval");
391 const approvalNodeId = NodeId.make("node:stalled-approval");
392 const approvalItemId = TurnItemId.make("turn-item:stalled-approval");
393 if (stalledRun === "returned-interrupt") {
394 yield* sink.write({
395 events: [
396 {
397 id: EventId.make("stalled-approval-node"),
398 type: "node.updated",
399 threadId,
400 runId: run.id,
401 occurredAt: now,
402 payload: {
403 ...node,
404 id: approvalNodeId,
405 parentNodeId: node.id,
406 kind: "approval_request",
407 status: "waiting",
408 countsForRun: false,
409 runtimeRequestId: approvalRequestId,
410 },
411 },
412 {
413 id: EventId.make("stalled-approval-request"),
414 type: "runtime-request.updated",
415 threadId,
416 occurredAt: now,
417 payload: {
418 id: approvalRequestId,
419 nodeId: approvalNodeId,
420 providerTurnId: codexTurn.id,
421 nativeRequestRef: { driver, nativeId: "approval", strength: "strong" },
422 kind: "command",
423 status: "pending",
424 responseCapability: {
425 type: "live",
426 providerSessionId: first.providerThread.providerSessionId!,
427 },
428 createdAt: now,
429 resolvedAt: null,
430 },
431 },
432 {
433 id: EventId.make("stalled-approval-item"),
434 type: "turn-item.updated",
435 threadId,
436 runId: run.id,
437 occurredAt: now,
438 payload: {
439 id: approvalItemId,
440 threadId,
441 runId: run.id,
442 nodeId: approvalNodeId,
443 providerThreadId: codexTurn.providerThreadId,
444 providerTurnId: codexTurn.id,
445 nativeItemRef: null,
446 parentItemId: null,
447 ordinal: 101,
448 status: "waiting",
449 title: null,
450 startedAt: now,
451 completedAt: null,
452 updatedAt: now,
453 type: "approval_request",
454 requestId: approvalRequestId,
455 requestKind: "command",
456 prompt: "rm -rf build",
457 },
458 },
459 ],
460 });
461 }
462 if (stalledRun === "superseded-attempt") {
463 yield* sink.write({
464 events: [
465 {
466 id: EventId.make("new-attempt"),
467 type: "run.updated",
468 threadId,
469 occurredAt: now,
470 payload: { ...run, activeAttemptId: RunAttemptId.make("new-attempt") },
471 },
472 ],
473 });
474 }
475 yield* orchestrator.dispatch(
476 stalledRun === "superseded-attempt"
477 ? {
478 type: "thread.background-work.settle",
479 commandId: CommandId.make("late-settle"),
480 threadId,
481 providerThreadId: codexTurn.providerThreadId,
482 providerTurnId: codexTurn.id,
483 }
484 : {
485 type: "thread.stop",
486 commandId: CommandId.make("stop-stalled-run"),
487 threadId,
488 },
489 );
490 yield* stalledRun === "returned-interrupt" ? drainUnconfirmedStop : worker.drain();
491 const after = yield* orchestrator.getThreadProjection(threadId);
492 const interrupted = stalledRun !== "superseded-attempt";
493 assert.equal(after.runs[0]?.status, interrupted ? "interrupted" : "running");
494 assert.equal(after.attempts[0]?.status, interrupted ? "interrupted" : "running");
495 assert.equal(
496 after.providerTurns[0]?.status,
497 terminalProviderTurn ? "completed" : interrupted ? "interrupted" : "running",
498 );
499 assert.equal(
500 after.nodes.find((candidate) => candidate.id === node.id)?.status,
501 interrupted ? "interrupted" : "running",
502 );
503 const output = after.messages.find((message) => message.id === messageId)!;
504 assert.equal(output.streaming, !interrupted);
505 assert.equal(output.text, "Partial output");
506 assert.equal(
507 after.turnItems.find((candidate) => candidate.id === devServerId)?.status,
508 stalledRun === "missing-session-terminal"
509 ? "completed"
510 : interrupted
511 ? "interrupted"
512 : "running",
513 );
514 assert.equal(
515 after.turnItems.find((candidate) => candidate.type === "assistant_message")?.status,
516 interrupted ? "interrupted" : "running",
517 );
518 assert.equal(
519 after.turnItems.filter((candidate) => candidate.type === "run_interrupt_result").length,
520 interrupted ? 1 : 0,
521 );
522 assert.isEmpty(after.runs.filter((candidate) => candidate.status === "waiting"));
523 if (stalledRun === "returned-interrupt") {
524 assert.deepEqual(
525 [
526 after.runtimeRequests.find((candidate) => candidate.id === approvalRequestId)?.status,
527 after.nodes.find((candidate) => candidate.id === approvalNodeId)?.status,
528 after.turnItems.find((candidate) => candidate.id === approvalItemId)?.status,
529 ],
530 ["cancelled", "interrupted", "interrupted"],
531 );
532 }
533 return;
534 }
535 const settled = yield* watch(
536 (event) =>
537 event.type === "run.updated" &&
538 event.payload.id === first.runId &&
539 event.payload.status === "waiting",
540 );
541 yield* Queue.offer(events, {
542 type: "provider_turn.updated",
543 driver,
544 providerTurn: { ...codexTurn, status: "completed", completedAt: now },
545 });
546 yield* Queue.offer(events, {
547 type: "turn.terminal",
548 driver,
549 providerThreadId: codexTurn.providerThreadId,
550 providerTurnId: codexTurn.id,
551 runOrdinal: first.runOrdinal,
552 status: "completed",
553 failure: null,
554 threadDisposition: "reusable",
555 });
556 yield* Fiber.join(settled);
557 yield* worker.drain();
558
559 const codexProviderThread = (yield* orchestrator.getThreadProjection(threadId))
560 .providerThreads[0]!;
561 const backgroundTasks = [
562 { taskId: "sleep", description: "Wait before the recheck", kind: "command" as const },
563 ];
564 const otherBackgroundTasks = [{ ...backgroundTasks[0]!, taskId: "other-sleep" }];
565 const subagentThread = (name: string) =>
566 Effect.gen(function* () {
567 const subagentThreadId = ThreadId.make(`thread:${name}`);
568 const rootId = NodeId.make(`node:${name}-root`);
569 yield* orchestrator.dispatch({
570 type: "thread.create",
571 commandId: CommandId.make(`create-${name}`),
572 threadId: subagentThreadId,
573 projectId: ProjectId.make("project:background-work-stop"),
574 title: name,
575 modelSelection,
576 runtimeMode: "full-access",
577 interactionMode: "default",
578 branch: null,
579 worktreePath: cwd,
580 createdBy: "agent",
581 creationSource: "provider",
582 });
583 yield* sink.write({
584 events: [
585 {
586 id: EventId.make(`${name}-root`),
587 type: "node.updated",
588 threadId: subagentThreadId,
589 occurredAt: now,
590 payload: {
591 id: rootId,
592 threadId: subagentThreadId,
593 runId: null,
594 parentNodeId: null,
595 rootNodeId: rootId,
596 kind: "root_turn",
597 status: "running",
598 countsForRun: false,
599 providerThreadId: null,
600 providerTurnId: null,
601 nativeItemRef: null,
602 runtimeRequestId: null,
603 checkpointScopeId: null,
604 startedAt: now,
605 completedAt: null,
606 },
607 },
608 ],
609 });
610 return { threadId: subagentThreadId, rootId };
611 });
612 const { threadId: reviewerThreadId, rootId: reviewerRootId } =
613 yield* subagentThread("reviewer");
614 const tester = yield* subagentThread("tester");
615 const nestedSubagentId = NodeId.make("subagent:nested");
616 const nestedItemId = TurnItemId.make("turn-item:nested-subagent");
617 const nested = {
618 threadId: reviewerThreadId,
619 runId: null,
620 origin: "provider_native",
621 driver,
622 providerInstanceId: instanceId,
623 providerThreadId: null,
624 childThreadId: tester.threadId,
625 prompt: "Check the tests",
626 title: null,
627 status: "running",
628 result: null,
629 startedAt: now,
630 completedAt: null,
631 updatedAt: now,
632 } as const;
633 yield* sink.write({
634 events: [
635 ...(workflowMember
636 ? []
637 : [
638 {
639 id: EventId.make("nested-subagent-item"),
640 type: "turn-item.updated",
641 threadId: reviewerThreadId,
642 occurredAt: now,
643 payload: {
644 ...nested,
645 id: nestedItemId,
646 nodeId: nestedSubagentId,
647 providerTurnId: null,
648 nativeItemRef: null,
649 parentItemId: null,
650 ordinal: 1,
651 type: "subagent",
652 subagentId: nestedSubagentId,
653 },
654 } as const,
655 ]),
656 {
657 id: EventId.make("nested-subagent"),
658 type: "subagent.updated",
659 threadId: reviewerThreadId,
660 occurredAt: now,
661 payload: {
662 ...nested,
663 id: nestedSubagentId,
664 parentNodeId: reviewerRootId,
665 createdBy: "agent",
666 nativeTaskRef: workflowMember
667 ? { driver, nativeId: "workflow:member", strength: "strong" }
668 : null,
669 model: null,
670 },
671 },
672 ],
673 });
674 if (workflowMember) {
675 const memberRoot = (yield* orchestrator.getThreadProjection(tester.threadId)).nodes[0]!;
676 yield* sink.write({
677 events: [
678 {
679 id: EventId.make("workflow-member-node"),
680 type: "node.updated",
681 threadId: reviewerThreadId,
682 occurredAt: now,
683 payload: {
684 ...memberRoot,
685 id: nestedSubagentId,
686 threadId: reviewerThreadId,
687 parentNodeId: reviewerRootId,
688 rootNodeId: reviewerRootId,
689 kind: "subagent",
690 },
691 },
692 ],
693 });
694 }
695 const settledRun = (input: {
696 readonly ordinal: number;
697 readonly providerThreadId: ProviderThreadId;
698 readonly runningItem?: {
699 readonly id: TurnItemId;
700 readonly kind: "command" | "subagent";
701 };
702 }) => {
703 const runId = RunId.make(`run:${input.ordinal}`);
704 const attemptId = RunAttemptId.make(`attempt:${input.ordinal}`);
705 const nodeId = NodeId.make(`node:${input.ordinal}`);
706 const providerTurnId = ProviderTurnId.make(`provider-turn:${input.ordinal}`);
707 const events: Array<OrchestrationV2DomainEvent> = [
708 {
709 id: EventId.make(`run:${input.ordinal}`),
710 type: "run.created",
711 threadId,
712 occurredAt: now,
713 payload: {
714 id: runId,
715 threadId,
716 ordinal: input.ordinal,
717 providerInstanceId: instanceId,
718 modelSelection,
719 providerThreadId: input.providerThreadId,
720 userMessageId: MessageId.make(`message:${input.ordinal}`),
721 rootNodeId: nodeId,
722 activeAttemptId: attemptId,
723 status: "completed",
724 requestedAt: now,
725 startedAt: now,
726 completedAt: now,
727 checkpointId: null,
728 contextHandoffId: null,
729 },
730 },
731 {
732 id: EventId.make(`node:${input.ordinal}`),
733 type: "node.updated",
734 threadId,
735 runId,
736 occurredAt: now,
737 payload: {
738 id: nodeId,
739 threadId,
740 runId,
741 parentNodeId: null,
742 rootNodeId: nodeId,
743 kind: "root_turn",
744 status: "completed",
745 countsForRun: true,
746 providerThreadId: input.providerThreadId,
747 providerTurnId,
748 nativeItemRef: null,
749 runtimeRequestId: null,
750 checkpointScopeId: null,
751 startedAt: now,
752 completedAt: now,
753 },
754 },
755 {
756 id: EventId.make(`attempt:${input.ordinal}`),
757 type: "run-attempt.created",
758 threadId,
759 occurredAt: now,
760 payload: {
761 id: attemptId,
762 runId,
763 attemptOrdinal: 1,
764 rootNodeId: nodeId,
765 providerInstanceId: instanceId,
766 providerThreadId: input.providerThreadId,
767 providerTurnId,
768 reason: "initial",
769 status: "completed",
770 startedAt: now,
771 completedAt: now,
772 },
773 },
774 {
775 id: EventId.make(`provider-turn:${input.ordinal}`),
776 type: "provider-turn.updated",
777 threadId,
778 occurredAt: now,
779 payload: {
780 id: providerTurnId,
781 providerThreadId: input.providerThreadId,
782 nodeId,
783 runAttemptId: attemptId,
784 nativeTurnRef: null,
785 ordinal: input.ordinal,
786 status: "completed",
787 startedAt: now,
788 completedAt: now,
789 },
790 },
791 ];
792 const item = input.runningItem;
793 if (item !== undefined) {
794 const base = {
795 id: item.id,
796 threadId,
797 runId,
798 nodeId,
799 providerThreadId: input.providerThreadId,
800 providerTurnId,
801 nativeItemRef: null,
802 parentItemId: null,
803 ordinal: input.ordinal * 100,
804 status: "running",
805 title: null,
806 startedAt: now,
807 completedAt: null,
808 updatedAt: now,
809 } as const;
810 events.push({
811 id: EventId.make(`item:${input.ordinal}`),
812 type: "turn-item.updated",
813 threadId,
814 runId,
815 occurredAt: now,
816 payload:
817 item.kind === "command"
818 ? { ...base, type: "command_execution", input: "vp run test --watch" }
819 : {
820 ...base,
821 providerThreadId: ProviderThreadId.make("provider-thread:codex-subagent"),
822 type: "subagent",
823 subagentId: NodeId.make(`subagent:${input.ordinal}`),
824 origin: "provider_native",
825 driver,
826 providerInstanceId: instanceId,
827 childThreadId: reviewerThreadId,
828 prompt: "Review the change",
829 result: null,
830 },
831 });
832 if (item.kind === "subagent") {
833 events.push({
834 id: EventId.make(`subagent:${input.ordinal}`),
835 type: "subagent.updated",
836 threadId,
837 runId,
838 occurredAt: now,
839 payload: {
840 ...base,
841 id: NodeId.make(`subagent:${input.ordinal}`),
842 parentNodeId: nodeId,
843 origin: "provider_native",
844 createdBy: "agent",
845 driver,
846 providerInstanceId: instanceId,
847 providerThreadId: null,
848 childThreadId: reviewerThreadId,
849 nativeTaskRef: null,
850 prompt: "Review the change",
851 model: null,
852 result: null,
853 },
854 });
855 }
856 }
857 return { runId, providerTurnId, events };
858 };
859
860 const watcherId = TurnItemId.make("turn-item:watcher");
861 const otherProviderThreadId = ProviderThreadId.make("provider-thread:other");
862 const watcherRun = settledRun({
863 ordinal: 2,
864 providerThreadId: olderStart ? otherProviderThreadId : codexProviderThread.id,
865 ...(olderStart ? {} : { runningItem: { id: watcherId, kind: "command" as const } }),
866 });
867 const reviewerId = TurnItemId.make("turn-item:reviewer");
868 const reviewerRun = settledRun({
869 ordinal: 3,
870 providerThreadId: codexProviderThread.id,
871 runningItem: { id: reviewerId, kind: "subagent" },
872 });
873 const latestRun = settledRun({ ordinal: 4, providerThreadId: otherProviderThreadId });
874 yield* sink.write({
875 events: [
876 ...watcherRun.events,
877 ...reviewerRun.events,
878 {
879 id: EventId.make("provider-thread:other"),
880 type: "provider-thread.updated",
881 threadId,
882 occurredAt: now,
883 payload: {
884 ...codexProviderThread,
885 id: otherProviderThreadId,
886 firstRunOrdinal: 4,
887 lastRunOrdinal: 4,
888 ...(failedStart ? { pendingBackgroundTasks: otherBackgroundTasks } : {}),
889 },
890 },
891 ...latestRun.events,
892 ],
893 });
894
895 const failedRun = settledRun({ ordinal: 5, providerThreadId: codexProviderThread.id });
896 if (failedStart) {
897 yield* sink.write({
898 events: [
899 {
900 id: EventId.make("failed-start-roster"),
901 type: "provider-thread.updated",
902 threadId,
903 occurredAt: now,
904 payload: {
905 ...codexProviderThread,
906 lastRunOrdinal: 5,
907 pendingBackgroundTasks: backgroundTasks,
908 },
909 },
910 ...failedRun.events.flatMap((event): Array<OrchestrationV2DomainEvent> => {
911 switch (event.type) {
912 case "provider-turn.updated":
913 return [];
914 case "run.created":
915 return [{ ...event, payload: { ...event.payload, status: "failed" } }];
916 case "node.updated":
917 return [
918 {
919 ...event,
920 payload: { ...event.payload, status: "failed", providerTurnId: null },
921 },
922 ];
923 case "run-attempt.created":
924 return [
925 {
926 ...event,
927 payload: { ...event.payload, status: "failed", providerTurnId: null },
928 },
929 ];
930 default:
931 return [event];
932 }
933 }),
934 ],
935 });
936 // The older turn reports an empty roster, but the refused run owns the row now.
937 const clear = yield* sink.writeIfProviderThreadOwner({
938 providerThreadId: codexProviderThread.id,
939 runId: reviewerRun.runId,
940 activeAttemptId: RunAttemptId.make("attempt:3"),
941 expectedLastRunOrdinal: 3,
942 events: [
943 {
944 id: EventId.make("dropped-roster-clear"),
945 type: "provider-thread.updated",
946 threadId,
947 occurredAt: now,
948 payload: { ...codexProviderThread, pendingBackgroundTasks: [] },
949 },
950 ],
951 });
952 assert.isFalse(clear.committed);
953 const beforeStop = (yield* orchestrator.getThreadProjection(threadId)).providerThreads.find(
954 (thread) => thread.id === codexProviderThread.id,
955 )!;
956 assert.equal(beforeStop.lastRunOrdinal, 5);
957 assert.deepEqual(beforeStop.pendingBackgroundTasks, backgroundTasks);
958 }
959
960 if (stopWithQueue !== undefined) {
961 const owner = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
962 (run) => run.id === latestRun.runId,
963 )!;
964 yield* sink.write({
965 events: [
966 {
967 id: EventId.make("latest-run-waiting"),
968 type: "run.updated",
969 threadId,
970 runId: owner.id,
971 occurredAt: now,
972 payload: { ...owner, status: "waiting", completedAt: null },
973 },
974 ],
975 });
976 yield* orchestrator.dispatch({
977 type: "message.dispatch",
978 commandId: CommandId.make("queue-follow-up"),
979 threadId,
980 messageId: MessageId.make("message:queue-follow-up"),
981 text: "Follow up after the background work",
982 attachments: [],
983 dispatchMode: { type: "queue_after_active" },
984 createdBy: "user",
985 creationSource: "web",
986 });
987 const queued = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!;
988 assert.equal(queued.status, "queued");
989 yield* sink.write({
990 events: [
991 {
992 id: EventId.make("queue-held"),
993 type: "run.updated",
994 threadId,
995 runId: queued.id,
996 occurredAt: now,
997 payload: { ...queued, queueHeld: true },
998 },
999 {
1000 id: EventId.make("latest-run-settled"),
1001 type: "run.updated",
1002 threadId,
1003 runId: owner.id,
1004 occurredAt: now,
1005 payload: owner,
1006 },
1007 ],
1008 });
1009 }
1010
1011 if (olderStart) {
1012 const older = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1013 (run) => run.id === watcherRun.runId,
1014 )!;
1015 yield* sink.write({
1016 events: [
1017 {
1018 id: EventId.make("older-run-starting"),
1019 type: "run.updated",
1020 threadId,
1021 runId: older.id,
1022 occurredAt: now,
1023 payload: { ...older, status: "starting", startedAt: null, completedAt: null },
1024 },
1025 ],
1026 });
1027 }
1028
1029 yield* orchestrator.dispatch(
1030 stopWithQueue === "thread.stop"
1031 ? {
1032 type: "thread.stop",
1033 commandId: CommandId.make("stop-background-work"),
1034 threadId,
1035 }
1036 : {
1037 type: "run.interrupt",
1038 commandId: CommandId.make("stop-background-work"),
1039 threadId,
1040 runId: olderStart
1041 ? watcherRun.runId
1042 : failedStart
1043 ? failedRun.runId
1044 : latestRun.runId,
1045 ...(stopWithQueue === undefined ? {} : { holdQueue: true }),
1046 },
1047 );
1048 yield* worker.drain();
1049
1050 assert.sameDeepMembers(
1051 interrupts.map((interrupt) => [interrupt.providerThread.id, interrupt.providerTurnId]),
1052 [
1053 ...(olderStart || failedStart ? [] : [[otherProviderThreadId, latestRun.providerTurnId]]),
1054 [codexProviderThread.id, reviewerRun.providerTurnId],
1055 ],
1056 );
1057 const after = yield* orchestrator.getThreadProjection(threadId);
1058 if (olderStart) {
1059 assert.equal(after.runs.find((run) => run.id === watcherRun.runId)?.status, "interrupted");
1060 }
1061 if (stopWithQueue !== undefined) {
1062 assert.equal(after.runs.at(-1)?.status, "queued");
1063 assert.equal(after.runs.at(-1)?.queueHeld, true);
1064 assert.lengthOf(started, 1);
1065 }
1066 assert.deepEqual(
1067 [devServerId, ...(olderStart ? [] : [watcherId]), reviewerId].map(
1068 (id) => after.turnItems.find((item) => item.id === id)?.status,
1069 ),
1070 olderStart ? ["interrupted", "interrupted"] : ["interrupted", "interrupted", "interrupted"],
1071 );
1072 assert.equal(after.subagents[0]?.status, "interrupted");
1073 const reviewerThread = yield* orchestrator.getThreadProjection(reviewerThreadId);
1074 assert.equal(reviewerThread.nodes[0]?.status, "interrupted");
1075 assert.equal(reviewerThread.subagents[0]?.status, "interrupted");
1076 if (workflowMember) {
1077 assert.isUndefined(reviewerThread.turnItems.find((item) => item.id === nestedItemId));
1078 assert.equal(
1079 reviewerThread.nodes.find((node) => node.id === nestedSubagentId)?.status,
1080 "interrupted",
1081 );
1082 } else {
1083 assert.equal(
1084 reviewerThread.turnItems.find((item) => item.id === nestedItemId)?.status,
1085 "interrupted",
1086 );
1087 }
1088 const testerThread = yield* orchestrator.getThreadProjection(tester.threadId);
1089 assert.equal(testerThread.nodes[0]?.status, "interrupted");
1090 if (failedStart) {
1091 assert.deepEqual(
1092 after.providerThreads.find((candidate) => candidate.id === codexProviderThread.id)
1093 ?.pendingBackgroundTasks,
1094 backgroundWorkAfterStop === undefined ? [] : backgroundTasks,
1095 );
1096 assert.deepEqual(
1097 after.providerThreads.find((candidate) => candidate.id === otherProviderThreadId)
1098 ?.pendingBackgroundTasks,
1099 otherBackgroundTasks,
1100 );
1101 }
1102 }).pipe(
1103 Effect.provide(
1104 ProviderReplayHarness.layerWithRegistry(
1105 { name: "background-work-stop" },
1106 ProviderAdapterRegistry.layerSingle(adapter),
1107 { runEffectWorker: false },
1108 ),
1109 ),
1110 );
1111 });
1112
1113 it.effect("Stop reaches background work an earlier provider thread still runs", () =>
1114 stopEarlierBackgroundWork({}),
1115 );
1116
1117 it.effect("Stop settles workflow members without subagent turn items", () =>
1118 stopEarlierBackgroundWork({ workflowMember: true }),
1119 );
1120
1121 it.effect(
1122 "Stop clears a stale roster after a refused run takes ownership of the provider thread",
1123 () => stopEarlierBackgroundWork({ failedStart: true }),
1124 );
1125
1126 it.effect("Stop preserves the roster when the background work probe fails", () =>
1127 stopEarlierBackgroundWork({ failedStart: true, backgroundWorkAfterStop: "unknown" }),
1128 );
1129
1130 it.effect.each(["thread.stop", "run.interrupt"] as const)(
1131 "%s stops background work when a later message is queued",
1132 (stopType) => stopEarlierBackgroundWork({ stopWithQueue: stopType }),
1133 );
1134
1135 it.effect.each(["thread.stop", "run.interrupt"] as const)(
1136 "%s reaches later background work when an older run is starting",
1137 (stopType) => stopEarlierBackgroundWork({ stopWithQueue: stopType, olderStart: true }),
1138 );
1139
1140 it.effect.each(["missing-session-terminal", "returned-interrupt", "superseded-attempt"] as const)(
1141 "Stop recovers a stalled run after %s without changing a newer attempt",
1142 (stalledRun) => stopEarlierBackgroundWork({ stalledRun }),
1143 );
1144
1145 it.effect("Stop holds a provider wake that queues before the stopped turn ends", () =>
1146 stopEarlierBackgroundWork({ wakeAfterStop: true }),
1147 );