@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 2,243 lines 85.6 KB
Raw
1 import { assert, it } from "@effect/vitest";
2 import {
3 CommandId,
4 ContextTransferId,
5 EventId,
6 MessageId,
7 NodeId,
8 type OrchestrationV2DomainEvent,
9 ProjectId,
10 ProviderDriverKind,
11 ProviderInstanceId,
12 ProviderSessionId,
13 ProviderThreadId,
14 ProviderTurnId,
15 RuntimeRequestId,
16 ScheduledTaskId,
17 ThreadId,
18 TurnItemId,
19 } from "@t3tools/contracts";
20 import * as DateTime from "effect/DateTime";
21 import * as Effect from "effect/Effect";
22 import * as Exit from "effect/Exit";
23 import * as Fiber from "effect/Fiber";
24 import * as Layer from "effect/Layer";
25 import * as Queue from "effect/Queue";
26 import * as Schema from "effect/Schema";
27 import * as SqlClient from "effect/sql/SqlClient";
28 import * as Stream from "effect/Stream";
29 import * as EventStore from "./EventStore.ts";
30 import * as TestClock from "effect/testing/TestClock";
31
32 import * as SqlitePersistence from "../persistence/Sqlite.ts";
33 import * as ServerSettings from "../serverSettings.ts";
34 import { CLAUDE_PROVIDER } from "./Adapters/ClaudeAdapterV2.ts";
35 import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
36 import * as CommandReceiptStore from "./CommandReceiptStore.ts";
37 import { OrchestrationEffectRequestV2 } from "./EffectOutbox.ts";
38 import * as Orchestrator from "./Orchestrator.ts";
39 import * as EffectWorker from "./EffectWorker.ts";
40 import * as EffectOutbox from "./EffectOutbox.ts";
41 import * as ProjectionStore from "./ProjectionStore.ts";
42 import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts";
43 import * as ProviderSessionManager from "./ProviderSessionManager.ts";
44 import * as ThreadManagementService from "./ThreadManagementService.ts";
45 import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts";
46 import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace";
47 import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
48
49 const instanceId = ProviderInstanceId.make("codex");
50 const modelSelection = { instanceId, model: "gpt-5.1-codex" };
51 const adapter = {
52 instanceId,
53 driver: ProviderDriverKind.make("codex"),
54 getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2),
55 planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" as const }),
56 openSession: () => Effect.die("Runs here never reach a provider"),
57 } as ProviderAdapter.ProviderAdapterV2["Service"];
58 const layerDatabase = SqlitePersistence.layerMemory;
59 const layerTest = ThreadManagementService.layer.pipe(
60 Layer.provideMerge(
61 Layer.mergeAll(
62 layerDatabase,
63 ProjectionStore.layer.pipe(Layer.provide(layerDatabase)),
64 EventStore.layer.pipe(Layer.provide(layerDatabase)),
65 EffectOutbox.layer.pipe(Layer.provide(layerDatabase)),
66 ProviderReplayHarness.layerWithRegistry(
67 { name: "thread-stop" },
68 ProviderAdapterRegistry.layerFromAdapters([adapter]),
69 { databaseLayer: layerDatabase, runEffectWorker: false },
70 ),
71 ),
72 ),
73 Layer.provideMerge(ServerSettings.layerTest()),
74 Layer.provideMerge(CommandReceiptStore.layer.pipe(Layer.provide(layerDatabase))),
75 );
76
77 const encodeEffectRequest = Schema.encodeSync(Schema.fromJsonString(OrchestrationEffectRequestV2));
78
79 const pullRequest = (number: number) => ({
80 host: "github.com",
81 repository: "pingdotgg/t3code",
82 number,
83 });
84
85 const createWatchingThread = (threadId: ThreadId, number: number) =>
86 Effect.gen(function* () {
87 const orchestrator = yield* Orchestrator.OrchestratorV2;
88 yield* orchestrator.dispatch({
89 type: "thread.create",
90 commandId: CommandId.make(`create:${threadId}`),
91 threadId,
92 projectId: ProjectId.make("project:thread-stop"),
93 title: threadId,
94 modelSelection,
95 runtimeMode: "full-access",
96 interactionMode: "default",
97 branch: null,
98 worktreePath: null,
99 createdBy: "user",
100 creationSource: "web",
101 });
102 yield* watch(threadId, number);
103 });
104
105 const watch = (threadId: ThreadId, number: number) =>
106 Effect.gen(function* () {
107 const orchestrator = yield* Orchestrator.OrchestratorV2;
108 yield* orchestrator.dispatch({
109 type: "thread.pull-request.watch",
110 commandId: CommandId.make(`watch:${threadId}:${number}`),
111 threadId,
112 ...pullRequest(number),
113 watching: true,
114 link: { url: `https://github.com/pingdotgg/t3code/pull/${number}`, source: "agent" },
115 });
116 });
117
118 const send = (threadId: ThreadId, text: string, type: "start_immediately" | "queue_after_active") =>
119 Effect.gen(function* () {
120 const orchestrator = yield* Orchestrator.OrchestratorV2;
121 yield* orchestrator.dispatch({
122 type: "message.dispatch",
123 commandId: CommandId.make(`send:${threadId}:${text}`),
124 threadId,
125 messageId: MessageId.make(`message:${threadId}:${text}`),
126 text,
127 attachments: [],
128 dispatchMode: { type },
129 createdBy: "user",
130 creationSource: "web",
131 });
132 });
133
134 const delegate = (parentThreadId: ThreadId, task: string) =>
135 Effect.gen(function* () {
136 const orchestrator = yield* Orchestrator.OrchestratorV2;
137 const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.at(-1)!;
138 yield* orchestrator.dispatch({
139 type: "delegated_task.request",
140 commandId: CommandId.make(`delegate:${task}`),
141 parentThreadId,
142 parentRunId: parentRun.id,
143 parentNodeId: parentRun.rootNodeId!,
144 task,
145 modelSelection,
146 runtimeMode: "full-access",
147 interactionMode: "default",
148 completionWake: "always",
149 createdBy: "agent",
150 creationSource: "mcp",
151 });
152 const projection = yield* orchestrator.getThreadProjection(parentThreadId);
153 return projection.subagents.find((candidate) => candidate.prompt === task)!.childThreadId!;
154 });
155
156 const threadState = (threadId: ThreadId) =>
157 Effect.gen(function* () {
158 const orchestrator = yield* Orchestrator.OrchestratorV2;
159 const projection = yield* orchestrator.getThreadProjection(threadId);
160 return {
161 runs: projection.runs.map((run) => `${run.status}${run.queueHeld === true ? ":held" : ""}`),
162 watched: (projection.thread.pullRequests ?? [])
163 .filter((link) => link.watch !== undefined)
164 .map((link) => link.number),
165 };
166 });
167
168 it.effect.each([
169 ["idle", "thread.stop"],
170 ["idle", "run.interrupt"],
171 ["idle", "subagent.stop"],
172 ["running", "subagent.stop"],
173 ["bound", "subagent.stop"],
174 ["correlated", "subagent.stop"],
175 ["archived", "thread.stop"],
176 ["missing", "thread.stop"],
177 ["running", "thread.stop"],
178 ["queued", "thread.stop"],
179 ["waiting", "thread.stop"],
180 ["correlated", "thread.stop"],
181 ["provider-active", "thread.stop"],
182 ["roster", "thread.stop"],
183 ["background", "thread.stop"],
184 ["question", "thread.stop"],
185 ["restart", "thread.stop"],
186 ] as const)(
187 "Stop clears an orphaned follow-up when its child is finished; its own Stop does unless a run is bound (%s via %s)",
188 ([childState, type]) =>
189 Effect.gen(function* () {
190 const orchestrator = yield* Orchestrator.OrchestratorV2;
191 const projections = yield* ProjectionStore.ProjectionStoreV2;
192 const parentThreadId = ThreadId.make(`thread:orphan-parent:${childState}`);
193 yield* createWatchingThread(parentThreadId, 1);
194 yield* send(parentThreadId, "parent", "start_immediately");
195 const childThreadId = yield* delegate(parentThreadId, "original task");
196 const parent = yield* orchestrator.getThreadProjection(parentThreadId);
197 const child = yield* orchestrator.getThreadProjection(childThreadId);
198 const original = parent.subagents[0]!;
199 const originalItem = parent.turnItems.find((item) => item.type === "subagent")!;
200 const originalNode = parent.nodes.find((node) => node.id === original.id)!;
201 const taskId = NodeId.make(`node:orphan-followup:${childState}`);
202 const itemId = TurnItemId.make(`item:orphan-followup:${childState}`);
203 const followupChildThreadId =
204 childState === "missing" ? ThreadId.make("thread:missing-child") : childThreadId;
205 const now = yield* DateTime.now;
206 let sequence = 0;
207 const apply = (event: Omit<OrchestrationV2DomainEvent, "id">) =>
208 projections.apply({
209 ...event,
210 id: EventId.make(`event:orphan:${childState}:${sequence++}`),
211 } as OrchestrationV2DomainEvent);
212 yield* apply({
213 type: "subagent.updated",
214 threadId: parentThreadId,
215 occurredAt: now,
216 payload: {
217 ...original,
218 status: "completed",
219 result: "Original result",
220 completedAt: now,
221 completionDelivery: { state: "acknowledged", observedByRunId: null },
222 },
223 });
224 yield* apply({
225 type: "context-transfer.created",
226 threadId: parentThreadId,
227 occurredAt: now,
228 payload: {
229 ...child.contextTransfers[0]!,
230 id: ContextTransferId.make(`transfer:original-result:${childState}`),
231 type: "subagent_result",
232 sourceThreadId: childThreadId,
233 targetThreadId: parentThreadId,
234 sourcePoint: { threadId: childThreadId, runId: child.runs[0]!.id },
235 targetRunId: parent.runs[0]!.id,
236 },
237 });
238 yield* apply({
239 type: "node.updated",
240 threadId: parentThreadId,
241 occurredAt: now,
242 payload: { ...originalNode, status: "completed", completedAt: now },
243 });
244 yield* apply({
245 type: "turn-item.updated",
246 threadId: parentThreadId,
247 occurredAt: now,
248 payload: { ...originalItem, status: "completed", completedAt: now },
249 });
250 yield* apply({
251 type: "subagent.updated",
252 threadId: parentThreadId,
253 occurredAt: now,
254 payload: {
255 ...original,
256 id: taskId,
257 childThreadId: followupChildThreadId,
258 status: "running",
259 result: null,
260 completedAt: null,
261 },
262 });
263 yield* apply({
264 type: "node.updated",
265 threadId: parentThreadId,
266 occurredAt: now,
267 payload: { ...originalNode, id: taskId, status: "running" },
268 });
269 assert.equal(originalItem.type, "subagent");
270 if (originalItem.type !== "subagent") return;
271 yield* apply({
272 type: "turn-item.updated",
273 threadId: parentThreadId,
274 occurredAt: now,
275 payload: {
276 ...originalItem,
277 id: itemId,
278 nodeId: taskId,
279 subagentId: taskId,
280 childThreadId: followupChildThreadId,
281 status: "running",
282 },
283 });
284 const runAtStart = child.runs[0]!;
285 const childRunStatus =
286 childState === "running" || childState === "queued" || childState === "waiting"
287 ? childState
288 : childState === "bound"
289 ? "running"
290 : "completed";
291 yield* apply({
292 type: "run.updated",
293 threadId: childThreadId,
294 occurredAt: now,
295 payload: { ...runAtStart, status: "running", delegatedTaskId: taskId },
296 });
297 assert.equal(
298 (yield* orchestrator.getThreadProjection(childThreadId)).runs[0]?.delegatedTaskId,
299 taskId,
300 );
301 // A terminal write based on the attempt's start snapshot loses a later steer binding.
302 yield* apply({
303 type: "run.updated",
304 threadId: childThreadId,
305 occurredAt: now,
306 payload: {
307 ...runAtStart,
308 status: childRunStatus,
309 completedAt: childRunStatus === "completed" ? now : null,
310 ...(childState === "correlated" || childState === "bound"
311 ? { delegatedTaskId: taskId }
312 : {}),
313 },
314 });
315 assert.equal(
316 (yield* orchestrator.getThreadProjection(childThreadId)).runs[0]?.delegatedTaskId,
317 childState === "correlated" || childState === "bound" ? taskId : undefined,
318 );
319 if (childState === "provider-active" || childState === "roster") {
320 yield* apply({
321 type: "provider-thread.updated",
322 threadId: childThreadId,
323 occurredAt: now,
324 payload: {
325 ...child.providerThreads[0]!,
326 status: childState === "provider-active" ? "active" : "idle",
327 pendingBackgroundTasks:
328 childState === "roster" ? [{ taskId: "background-task", kind: "command" }] : [],
329 },
330 });
331 }
332 if (childState === "background") {
333 yield* apply({
334 type: "turn-item.updated",
335 threadId: childThreadId,
336 occurredAt: now,
337 payload: {
338 ...originalItem,
339 id: TurnItemId.make("item:child-background"),
340 threadId: childThreadId,
341 runId: runAtStart.id,
342 nodeId: runAtStart.rootNodeId,
343 type: "command_execution",
344 status: "running",
345 input: "test background work",
346 },
347 });
348 }
349 if (childState === "question") {
350 yield* apply({
351 type: "runtime-request.updated",
352 threadId: childThreadId,
353 occurredAt: now,
354 payload: {
355 id: RuntimeRequestId.make("request:child-question"),
356 nodeId: runAtStart.rootNodeId!,
357 providerTurnId: null,
358 nativeRequestRef: null,
359 kind: "user_input",
360 status: "pending",
361 responseCapability: { type: "message" },
362 createdAt: now,
363 resolvedAt: null,
364 },
365 });
366 }
367 if (childState === "restart") {
368 const outbox = yield* EffectOutbox.EffectOutboxV2;
369 yield* outbox.enqueue([
370 {
371 id: `effect:restart-continuation:${runAtStart.id}`,
372 commandId: CommandId.make("command:child-restart"),
373 threadId: childThreadId,
374 request: { type: "provider-runtime.continue", sourceRunId: runAtStart.id },
375 },
376 ]);
377 }
378 yield* apply({
379 type: "run.updated",
380 threadId: parentThreadId,
381 occurredAt: now,
382 payload: { ...parent.runs[0]!, status: "completed", completedAt: now },
383 });
384 if (type === "run.interrupt") {
385 const run = parent.runs[0]!;
386 yield* apply({
387 type: "provider-turn.updated",
388 threadId: parentThreadId,
389 occurredAt: now,
390 payload: {
391 id: ProviderTurnId.make("provider-turn:orphan-parent"),
392 providerThreadId: run.providerThreadId!,
393 nodeId: run.rootNodeId!,
394 runAttemptId: run.activeAttemptId!,
395 nativeTurnRef: null,
396 ordinal: 1,
397 status: "completed",
398 startedAt: now,
399 completedAt: now,
400 },
401 });
402 yield* apply({
403 type: "run-attempt.updated",
404 threadId: parentThreadId,
405 occurredAt: now,
406 payload: { ...parent.attempts[0]!, status: "completed", completedAt: now },
407 });
408 yield* apply({
409 type: "node.updated",
410 threadId: parentThreadId,
411 occurredAt: now,
412 payload: {
413 ...parent.nodes.find((node) => node.id === run.rootNodeId)!,
414 status: "completed",
415 completedAt: now,
416 },
417 });
418 }
419 if (childState === "archived") {
420 yield* apply({
421 type: "thread.archived",
422 threadId: childThreadId,
423 occurredAt: now,
424 payload: { ...child.thread, archivedAt: now },
425 });
426 }
427 const childBeforeStop = yield* orchestrator.getThreadProjection(childThreadId);
428 assert.lengthOf(
429 ((yield* orchestrator.getThreadShell(parentThreadId))?.pendingBackgroundTasks ?? []).filter(
430 (task) => task.kind === "subagent",
431 ),
432 1,
433 );
434 const stop = {
435 commandId: CommandId.make(`stop-orphan:${childState}`),
436 threadId: parentThreadId,
437 };
438 yield* orchestrator.dispatch(
439 type === "run.interrupt"
440 ? { ...stop, type, runId: parent.runs.at(-1)!.id, holdQueue: true }
441 : type === "subagent.stop"
442 ? { ...stop, type, runId: original.runId!, subagentId: taskId }
443 : { ...stop, type },
444 );
445 const after = yield* orchestrator.getThreadProjection(parentThreadId);
446 // The Lineage stop of one subagent leaves the parent's watches alone.
447 assert.lengthOf(
448 (after.thread.pullRequests ?? []).filter((link) => link.watch !== undefined),
449 type === "subagent.stop" ? 1 : 0,
450 );
451 // A task with a bound run, live or ended, keeps its settlement path.
452 const expected =
453 childState === "bound" || childState === "correlated"
454 ? "running"
455 : type === "subagent.stop" || childState === "idle" || childState === "archived"
456 ? "cancelled"
457 : "running";
458 assert.equal(after.subagents.find((task) => task.id === taskId)?.status, expected);
459 assert.equal(after.nodes.find((node) => node.id === taskId)?.status, expected);
460 assert.equal(after.turnItems.find((item) => item.id === itemId)?.status, expected);
461 assert.equal(
462 after.subagents.find((task) => task.id === taskId)?.completionDelivery?.state,
463 type === "subagent.stop" && expected === "running" ? undefined : "disposed",
464 );
465 assert.equal(
466 after.subagents.find((task) => task.id === original.id)?.result,
467 "Original result",
468 );
469 assert.deepEqual(yield* orchestrator.getThreadProjection(childThreadId), childBeforeStop);
470 if (expected === "cancelled") {
471 assert.lengthOf(
472 (
473 (yield* orchestrator.getThreadShell(parentThreadId))?.pendingBackgroundTasks ?? []
474 ).filter((task) => task.kind === "subagent"),
475 0,
476 );
477 yield* orchestrator.recoverDelegatedTasks;
478 yield* orchestrator.dispatch({
479 type: "thread.stop",
480 commandId: CommandId.make(`stop-orphan-again:${childState}`),
481 threadId: parentThreadId,
482 });
483 const repeated = yield* orchestrator.getThreadProjection(parentThreadId);
484 assert.equal(repeated.subagents.find((task) => task.id === taskId)?.status, "cancelled");
485 assert.deepEqual(repeated.messages, after.messages);
486 assert.deepEqual(repeated.contextTransfers, after.contextTransfers);
487 if (type === "subagent.stop") {
488 // A client that missed the cancel, such as one resumed from an older cache, still
489 // shows the task as running. Its Stop gets the ended task back.
490 const eventStore = yield* EventStore.EventStoreV2;
491 const afterSequence = yield* eventStore.latestSequence({ threadId: parentThreadId });
492 yield* orchestrator.dispatch({
493 ...stop,
494 commandId: CommandId.make(`stop-orphan-stale-client:${childState}`),
495 type,
496 runId: original.runId!,
497 subagentId: taskId,
498 });
499 const resent = yield* eventStore
500 .read({ threadId: parentThreadId, afterSequence })
501 .pipe(Stream.runCollect);
502 assert.deepEqual(
503 resent.flatMap(({ event }) =>
504 event.type === "subagent.updated" ? [event.payload.status] : [],
505 ),
506 ["cancelled"],
507 );
508 }
509 }
510 }).pipe(Effect.provide(layerTest)),
511 );
512
513 it.effect("Stop still stops the thread when a delegated child cannot be read", () =>
514 Effect.gen(function* () {
515 const orchestrator = yield* Orchestrator.OrchestratorV2;
516 const sql = yield* SqlClient.SqlClient;
517 const parentThreadId = ThreadId.make("thread:unreadable-child-parent");
518 yield* createWatchingThread(parentThreadId, 1);
519 yield* send(parentThreadId, "parent", "start_immediately");
520 const childThreadId = yield* delegate(parentThreadId, "unreadable child");
521 yield* sql`
522 UPDATE orchestration_v2_projection_runs
523 SET payload_json = '{"obsolete":true}'
524 WHERE thread_id = ${childThreadId}
525 `;
526 yield* orchestrator.dispatch({
527 type: "run.interrupt",
528 commandId: CommandId.make("stop:unreadable-child"),
529 threadId: parentThreadId,
530 runId: (yield* orchestrator.getThreadProjection(parentThreadId)).runs.at(-1)!.id,
531 holdQueue: true,
532 });
533 assert.deepEqual(yield* threadState(parentThreadId), { runs: ["interrupted"], watched: [] });
534 }).pipe(Effect.provide(layerTest)),
535 );
536
537 it.effect.each(["run.interrupt", "thread.stop"] as const)(
538 "%s Stop ends watches, holds queues, and stops the delegated tasks under the thread",
539 (type) =>
540 Effect.gen(function* () {
541 const orchestrator = yield* Orchestrator.OrchestratorV2;
542 const projections = yield* ProjectionStore.ProjectionStoreV2;
543 const threads = yield* ThreadManagementService.ThreadManagementService;
544 const sql = yield* SqlClient.SqlClient;
545 const parentThreadId = ThreadId.make("thread:stop-parent");
546 yield* createWatchingThread(parentThreadId, 1);
547 yield* send(parentThreadId, "first", "start_immediately");
548 const childThreadId = yield* delegate(parentThreadId, "child task");
549 assert.isTrue(Exit.isFailure(yield* Effect.exit(watch(childThreadId, 2))));
550 const grandchildThreadId = yield* delegate(childThreadId, "grandchild task");
551 yield* send(childThreadId, "child follow-up", "queue_after_active");
552
553 const now = yield* DateTime.now;
554 const firstRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs[0]!;
555 yield* projections.apply({
556 id: EventId.make("event:stop-parent:first-run-completed"),
557 type: "run.updated",
558 threadId: parentThreadId,
559 runId: firstRun.id,
560 occurredAt: now,
561 payload: { ...firstRun, status: "completed", completedAt: now },
562 });
563 yield* send(parentThreadId, "second", "start_immediately");
564 const secondRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.at(-1)!;
565
566 const stopCommandId = CommandId.make("stop-parent");
567 yield* orchestrator.dispatch(
568 type === "thread.stop"
569 ? { type, commandId: stopCommandId, threadId: parentThreadId }
570 : {
571 type,
572 commandId: stopCommandId,
573 threadId: parentThreadId,
574 runId: secondRun.id,
575 holdQueue: true,
576 },
577 );
578
579 assert.deepEqual(yield* threadState(parentThreadId), {
580 runs: ["completed", "interrupted"],
581 watched: [],
582 });
583 const parent = yield* orchestrator.getThreadProjection(parentThreadId);
584 assert.equal(parent.subagents[0]?.completionDelivery?.state, "disposed");
585 const effects = yield* sql<{ readonly effect_type: string }>`
586 SELECT effect_type FROM orchestration_v2_effect_outbox WHERE command_id = ${stopCommandId}
587 `;
588 assert.include(
589 effects.map((row) => row.effect_type),
590 "delegated-tasks.stop",
591 );
592
593 yield* threads.stopDelegatedTasks({ threadId: parentThreadId, commandId: stopCommandId });
594 assert.deepEqual(yield* threadState(childThreadId), {
595 runs: ["interrupted", "queued:held"],
596 watched: [],
597 });
598 assert.deepEqual((yield* threadState(grandchildThreadId)).runs, ["interrupted"]);
599
600 yield* threads.stopDelegatedTasks({ threadId: parentThreadId, commandId: stopCommandId });
601 assert.deepEqual((yield* threadState(childThreadId)).runs, ["interrupted", "queued:held"]);
602 // The walk stops each level itself, so the child stops it sends queue no walk of their own.
603 const walks = yield* sql<{ readonly command_id: string }>`
604 SELECT command_id FROM orchestration_v2_effect_outbox
605 WHERE effect_type = 'delegated-tasks.stop'
606 `;
607 assert.deepEqual(
608 walks.map((row) => row.command_id),
609 [stopCommandId],
610 );
611 }).pipe(Effect.provide(layerTest)),
612 );
613
614 it.effect("a repeated Stop that changes nothing still stops the delegated tasks", () =>
615 Effect.gen(function* () {
616 const orchestrator = yield* Orchestrator.OrchestratorV2;
617 const projections = yield* ProjectionStore.ProjectionStoreV2;
618 const sql = yield* SqlClient.SqlClient;
619 const parentThreadId = ThreadId.make("thread:stop-again");
620 yield* createWatchingThread(parentThreadId, 1);
621 yield* send(parentThreadId, "first", "start_immediately");
622 yield* delegate(parentThreadId, "stop-again child");
623 const now = yield* DateTime.now;
624 const run = (yield* orchestrator.getThreadProjection(parentThreadId)).runs[0]!;
625 yield* projections.apply({
626 id: EventId.make("event:stop-again:run-completed"),
627 type: "run.updated",
628 threadId: parentThreadId,
629 runId: run.id,
630 occurredAt: now,
631 payload: { ...run, status: "completed", completedAt: now },
632 });
633 yield* orchestrator.dispatch({
634 type: "thread.stop",
635 commandId: CommandId.make("stop-again:first"),
636 threadId: parentThreadId,
637 });
638
639 // The first Stop already disposed the delivery and ended the watch, so this one writes no
640 // events, like Stop on a thread an older server stopped without stopping its children.
641 const again = yield* orchestrator.dispatch({
642 type: "thread.stop",
643 commandId: CommandId.make("stop-again:second"),
644 threadId: parentThreadId,
645 });
646 assert.lengthOf(again.storedEvents, 0);
647 const walks = yield* sql<{ readonly command_id: string }>`
648 SELECT command_id FROM orchestration_v2_effect_outbox
649 WHERE effect_type = 'delegated-tasks.stop'
650 `;
651 assert.deepEqual(
652 walks.map((row) => row.command_id),
653 ["stop-again:first", "stop-again:second"],
654 );
655 }).pipe(Effect.provide(layerTest)),
656 );
657
658 it.effect.each(["thread.stop", "run.interrupt"] as const)(
659 "%s cancels buffered wakes and preserves durable queued messages",
660 (type) =>
661 Effect.gen(function* () {
662 const orchestrator = yield* Orchestrator.OrchestratorV2;
663 const threadId = ThreadId.make(`thread:stop-buffered:${type}`);
664 yield* createWatchingThread(threadId, 5);
665 yield* send(threadId, "active", "start_immediately");
666 yield* send(threadId, "user follow-up", "queue_after_active");
667 for (const creationSource of ["provider", "server"] as const) {
668 yield* orchestrator.dispatch({
669 type: "message.dispatch",
670 commandId: CommandId.make(`notification:${type}:${creationSource}`),
671 threadId,
672 messageId: MessageId.make(`notification:${type}:${creationSource}`),
673 text: "Background activity updated",
674 attachments: [],
675 createdBy: "agent",
676 creationSource,
677 notification: {
678 source: { kind: "background_task" },
679 outcome: "updated",
680 summary: "Background activity updated",
681 },
682 dispatchMode: { type: "queue_after_active" },
683 });
684 }
685 const before = yield* orchestrator.getThreadProjection(threadId);
686 const buffered = before.runs.find((run) =>
687 before.messages.some(
688 (message) => message.id === run.userMessageId && message.creationSource === "provider",
689 ),
690 )!;
691 const stop = { commandId: CommandId.make(`stop:${type}`), threadId };
692 yield* orchestrator.dispatch(
693 type === "run.interrupt"
694 ? { ...stop, type, runId: before.runs[0]!.id, holdQueue: true }
695 : { ...stop, type },
696 );
697 const stopped = yield* orchestrator.getThreadProjection(threadId);
698 assert.equal(stopped.runs.find((run) => run.id === buffered.id)?.status, "cancelled");
699 assert.equal(
700 stopped.attempts.find((attempt) => attempt.runId === buffered.id)?.status,
701 "cancelled",
702 );
703 assert.equal(
704 stopped.nodes.find((node) => node.id === buffered.rootNodeId)?.status,
705 "cancelled",
706 );
707 assert.deepEqual(
708 stopped.runs.filter((run) => run.status === "queued").map((run) => run.queueHeld),
709 [true, true],
710 );
711 }).pipe(Effect.provide(layerTest)),
712 );
713
714 it.effect("a message the user sends after Stop releases the held queue; an agent's does not", () =>
715 Effect.gen(function* () {
716 const orchestrator = yield* Orchestrator.OrchestratorV2;
717 const projections = yield* ProjectionStore.ProjectionStoreV2;
718 const threadId = ThreadId.make("thread:stop-release");
719 const agentSend = (text: string) =>
720 orchestrator.dispatch({
721 type: "message.dispatch",
722 commandId: CommandId.make(`agent-send:${text}`),
723 threadId,
724 messageId: MessageId.make(`message:${threadId}:${text}`),
725 text,
726 attachments: [],
727 dispatchMode: { type: "queue_after_active" },
728 createdBy: "agent",
729 creationSource: "mcp",
730 });
731 yield* createWatchingThread(threadId, 6);
732 yield* send(threadId, "active", "start_immediately");
733 yield* send(threadId, "queued before the stop", "queue_after_active");
734 yield* orchestrator.dispatch({
735 type: "thread.stop",
736 commandId: CommandId.make("stop-release"),
737 threadId,
738 });
739 assert.deepEqual((yield* threadState(threadId)).runs, ["interrupted", "queued:held"]);
740
741 // An agent's message starts on the idle thread, but the hold is the user's.
742 yield* agentSend("agent nudge");
743 assert.deepEqual((yield* threadState(threadId)).runs, [
744 "interrupted",
745 "queued:held",
746 "starting",
747 ]);
748 const nudge = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!;
749 const now = yield* DateTime.now;
750 yield* projections.apply({
751 id: EventId.make("event:stop-release:nudge-completed"),
752 type: "run.updated",
753 threadId,
754 runId: nudge.id,
755 occurredAt: now,
756 payload: { ...nudge, status: "completed", completedAt: now },
757 });
758
759 // A scheduled task the user created is not the user coming back.
760 yield* orchestrator.dispatch({
761 type: "message.dispatch",
762 commandId: CommandId.make("scheduled-send"),
763 threadId,
764 messageId: MessageId.make(`message:${threadId}:scheduled`),
765 text: "Nightly check",
766 attachments: [],
767 dispatchMode: { type: "queue_after_active" },
768 createdBy: "user",
769 creationSource: "web",
770 scheduledTaskId: ScheduledTaskId.make("scheduled:nightly"),
771 });
772 const scheduled = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!;
773 assert.deepEqual((yield* threadState(threadId)).runs, [
774 "interrupted",
775 "queued:held",
776 "completed",
777 "starting",
778 ]);
779 yield* projections.apply({
780 id: EventId.make("event:stop-release:scheduled-completed"),
781 type: "run.updated",
782 threadId,
783 runId: scheduled.id,
784 occurredAt: now,
785 payload: { ...scheduled, status: "completed", completedAt: now },
786 });
787
788 // The user is back: the held message follows their run, and a wake that
789 // queues behind it is not held by the old Stop.
790 yield* send(threadId, "back", "start_immediately");
791 yield* agentSend("task finished");
792 assert.deepEqual((yield* threadState(threadId)).runs, [
793 "interrupted",
794 "queued",
795 "completed",
796 "completed",
797 "starting",
798 "queued",
799 ]);
800 }).pipe(Effect.provide(layerTest)),
801 );
802
803 it.effect(
804 "an agent's auto send queues behind a run that is still stopping instead of steering it",
805 () =>
806 Effect.gen(function* () {
807 const orchestrator = yield* Orchestrator.OrchestratorV2;
808 const projections = yield* ProjectionStore.ProjectionStoreV2;
809 const threads = yield* ThreadManagementService.ThreadManagementService;
810 const threadId = ThreadId.make("thread:stop-mcp-send");
811 yield* createWatchingThread(threadId, 6);
812 yield* send(threadId, "active", "start_immediately");
813 const initial = yield* orchestrator.getThreadProjection(threadId);
814 const run = initial.runs[0]!;
815 const providerSessionId = ProviderSessionId.make("session:stop-mcp-send");
816 const now = yield* DateTime.now;
817 yield* projections.apply({
818 id: EventId.make("event:stop-mcp-send:session"),
819 type: "provider-session.updated",
820 threadId,
821 occurredAt: now,
822 payload: {
823 id: providerSessionId,
824 driver: adapter.driver,
825 providerInstanceId: instanceId,
826 status: "ready",
827 cwd: "/repo",
828 model: modelSelection.model,
829 capabilities: CodexProviderCapabilitiesV2,
830 createdAt: now,
831 updatedAt: now,
832 lastError: null,
833 },
834 });
835 yield* projections.apply({
836 id: EventId.make("event:stop-mcp-send:provider-thread"),
837 type: "provider-thread.updated",
838 threadId,
839 occurredAt: now,
840 payload: { ...initial.providerThreads[0]!, providerSessionId },
841 });
842 yield* projections.apply({
843 id: EventId.make("event:stop-mcp-send:running"),
844 type: "run.updated",
845 threadId,
846 runId: run.id,
847 occurredAt: now,
848 payload: { ...run, status: "running", startedAt: now },
849 });
850 yield* projections.apply({
851 id: EventId.make("event:stop-mcp-send:turn"),
852 type: "provider-turn.updated",
853 threadId,
854 occurredAt: now,
855 payload: {
856 id: ProviderTurnId.make("provider-turn:stop-mcp-send"),
857 providerThreadId: run.providerThreadId!,
858 nodeId: run.rootNodeId!,
859 runAttemptId: run.activeAttemptId!,
860 nativeTurnRef: null,
861 ordinal: 1,
862 status: "running",
863 startedAt: now,
864 completedAt: null,
865 },
866 });
867 yield* send(threadId, "queued before the stop", "queue_after_active");
868 yield* orchestrator.dispatch({
869 type: "thread.stop",
870 commandId: CommandId.make("stop-mcp-send"),
871 threadId,
872 });
873 assert.deepEqual((yield* threadState(threadId)).runs, ["running", "queued:held"]);
874
875 const result = yield* threads.sendToThread({
876 projectId: ProjectId.make("project:thread-stop"),
877 commandId: CommandId.make("command:stop-mcp-send"),
878 threadId,
879 messageId: MessageId.make("message:stop-mcp-send"),
880 text: "Follow up after Stop",
881 attachments: [],
882 mode: "auto",
883 createdBy: "agent",
884 creationSource: "mcp",
885 });
886 assert.equal(result.delivery, "queued");
887 assert.deepEqual((yield* threadState(threadId)).runs, [
888 "running",
889 "queued:held",
890 "queued:held",
891 ]);
892 }).pipe(Effect.provide(layerTest)),
893 );
894
895 it.effect(
896 "thread.stop ends an idle thread's watches and accepts a thread with nothing to stop",
897 () =>
898 Effect.gen(function* () {
899 const orchestrator = yield* Orchestrator.OrchestratorV2;
900 const threadId = ThreadId.make("thread:stop-idle");
901 yield* createWatchingThread(threadId, 3);
902
903 yield* orchestrator.dispatch({
904 type: "thread.stop",
905 commandId: CommandId.make("stop-idle"),
906 threadId,
907 });
908 assert.deepEqual(yield* threadState(threadId), { runs: [], watched: [] });
909
910 const again = yield* orchestrator.dispatch({
911 type: "thread.stop",
912 commandId: CommandId.make("stop-idle-again"),
913 threadId,
914 });
915 assert.lengthOf(again.storedEvents, 0);
916 }).pipe(Effect.provide(layerTest)),
917 );
918
919 it.effect.each([false, true])(
920 "a completed watch records whether it ended through Stop (%s)",
921 (stopped) =>
922 Effect.gen(function* () {
923 const orchestrator = yield* Orchestrator.OrchestratorV2;
924 const projections = yield* ProjectionStore.ProjectionStoreV2;
925 const threadId = ThreadId.make(`thread:watch-completion:${stopped}`);
926 yield* createWatchingThread(threadId, 7);
927 yield* send(threadId, "watch", "start_immediately");
928 const run = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!;
929 const now = yield* DateTime.now;
930 const completedRun = {
931 ...run,
932 providerThreadId: run.providerThreadId ?? ProviderThreadId.make("provider-thread:watch"),
933 status: "completed" as const,
934 startedAt: now,
935 completedAt: now,
936 };
937 yield* projections.apply({
938 id: EventId.make("event:watch-completed"),
939 type: "run.updated",
940 threadId,
941 occurredAt: now,
942 payload: completedRun,
943 });
944 assert.isFalse((yield* projections.getThreadShell(threadId))?.latestRunStopRequested);
945
946 yield* orchestrator.dispatch(
947 stopped
948 ? { type: "thread.stop", commandId: CommandId.make("stop-watch"), threadId }
949 : {
950 type: "thread.pull-request.watch",
951 commandId: CommandId.make("finish-watch"),
952 threadId,
953 ...pullRequest(7),
954 watching: false,
955 },
956 );
957 const shell = yield* projections.getThreadShell(threadId);
958 const snapshot = (yield* projections.getShellSnapshot()).threads.find(
959 (thread) => thread.id === threadId,
960 );
961 const memoryShell = ProjectionStore.threadShellFromProjection(
962 yield* orchestrator.getThreadProjection(threadId),
963 );
964 for (const result of [shell, snapshot, memoryShell]) {
965 assert.equal(result?.status, "completed");
966 assert.isEmpty(result?.pendingBackgroundTasks ?? []);
967 assert.equal(result?.latestRunStopRequested, stopped);
968 }
969
970 yield* send(threadId, "next", "start_immediately");
971 assert.isFalse((yield* projections.getThreadShell(threadId))?.latestRunStopRequested);
972 }).pipe(Effect.provide(layerTest)),
973 );
974
975 it.effect("a run Stop reached cannot delegate or start a watch, even after it ends", () =>
976 Effect.gen(function* () {
977 const orchestrator = yield* Orchestrator.OrchestratorV2;
978 const projections = yield* ProjectionStore.ProjectionStoreV2;
979 const threadId = ThreadId.make("thread:stop-barrier");
980 yield* createWatchingThread(threadId, 4);
981 yield* send(threadId, "work", "start_immediately");
982 const now = yield* DateTime.now;
983 const run = { ...(yield* orchestrator.getThreadProjection(threadId)).runs[0]!, startedAt: now };
984 yield* projections.apply({
985 id: EventId.make("event:stop-barrier:running"),
986 type: "run.updated",
987 threadId,
988 runId: run.id,
989 occurredAt: now,
990 payload: { ...run, status: "running" },
991 });
992 yield* send(threadId, "queued", "queue_after_active");
993 const queued = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!;
994 yield* orchestrator.dispatch({
995 type: "queued-run.cancel",
996 commandId: CommandId.make("cancel-queued"),
997 threadId,
998 runId: queued.id,
999 });
1000 yield* projections.apply({
1001 id: EventId.make("event:stop-barrier:interrupt-request"),
1002 type: "turn-item.updated",
1003 threadId,
1004 runId: run.id,
1005 occurredAt: now,
1006 payload: {
1007 id: TurnItemId.make("turn-item:stop-barrier:interrupt-request"),
1008 threadId,
1009 runId: run.id,
1010 nodeId: run.rootNodeId!,
1011 providerThreadId: run.providerThreadId!,
1012 providerTurnId: null,
1013 nativeItemRef: null,
1014 parentItemId: null,
1015 ordinal: 1_000,
1016 status: "completed",
1017 title: "Interrupt requested",
1018 startedAt: now,
1019 completedAt: now,
1020 updatedAt: now,
1021 type: "run_interrupt_request",
1022 message: "Stop",
1023 },
1024 });
1025
1026 assert.isTrue(Exit.isFailure(yield* Effect.exit(delegate(threadId, "late task"))));
1027 assert.isTrue(Exit.isFailure(yield* Effect.exit(watch(threadId, 5))));
1028
1029 yield* projections.apply({
1030 id: EventId.make("event:stop-barrier:interrupted"),
1031 type: "run.updated",
1032 threadId,
1033 runId: run.id,
1034 occurredAt: now,
1035 payload: { ...run, status: "interrupted", completedAt: now },
1036 });
1037 assert.isTrue(Exit.isFailure(yield* Effect.exit(watch(threadId, 5))));
1038 assert.deepEqual(yield* threadState(threadId), {
1039 runs: ["interrupted", "cancelled"],
1040 watched: [4],
1041 });
1042
1043 for (const watching of [false, true]) {
1044 yield* orchestrator.dispatch({
1045 type: "thread.pull-request.watch",
1046 commandId: CommandId.make(`manual-watch:${watching}`),
1047 threadId,
1048 ...pullRequest(4),
1049 watching,
1050 });
1051 }
1052 assert.deepEqual((yield* threadState(threadId)).watched, [4]);
1053 }).pipe(Effect.provide(layerTest)),
1054 );
1055
1056 it.effect("thread.stop keeps a restart continuation of the stopped run from starting", () =>
1057 Effect.gen(function* () {
1058 const orchestrator = yield* Orchestrator.OrchestratorV2;
1059 const projections = yield* ProjectionStore.ProjectionStoreV2;
1060 const sql = yield* SqlClient.SqlClient;
1061 const threadId = ThreadId.make("thread:stop-restart");
1062 yield* createWatchingThread(threadId, 8);
1063 yield* send(threadId, "work", "start_immediately");
1064 const now = yield* DateTime.now;
1065 const run = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!;
1066 yield* projections.apply({
1067 id: EventId.make("event:stop-restart:cancelled"),
1068 type: "run.updated",
1069 threadId,
1070 runId: run.id,
1071 occurredAt: now,
1072 payload: { ...run, status: "cancelled", startedAt: null, completedAt: now },
1073 });
1074 const at = DateTime.formatIso(now);
1075 yield* sql`
1076 INSERT INTO orchestration_v2_effect_outbox (
1077 effect_id, command_id, thread_id, effect_type, payload_json, status,
1078 attempt_count, available_at, created_at, updated_at
1079 ) VALUES (
1080 ${`effect:restart-continuation:${run.id}`}, 'command:restart', ${threadId},
1081 'provider-runtime.continue',
1082 ${encodeEffectRequest({ type: "provider-runtime.continue", sourceRunId: run.id })},
1083 'pending', 0, ${at}, ${at}, ${at}
1084 )
1085 `;
1086
1087 yield* orchestrator.dispatch({
1088 type: "thread.stop",
1089 commandId: CommandId.make("stop-restart"),
1090 threadId,
1091 });
1092 yield* orchestrator.dispatch({
1093 type: "message.dispatch",
1094 commandId: CommandId.make("restart-continuation"),
1095 threadId,
1096 messageId: MessageId.make("message:restart-continuation"),
1097 text: "Continue.",
1098 attachments: [],
1099 dispatchMode: { type: "start_immediately" },
1100 createdBy: "agent",
1101 creationSource: "server",
1102 restartContinuationOfRunId: run.id,
1103 });
1104 assert.deepEqual(yield* threadState(threadId), { runs: ["cancelled"], watched: [] });
1105 }).pipe(Effect.provide(layerTest)),
1106 );
1107
1108 it.effect.each(["thread.stop", "run.interrupt"] as const)(
1109 "a successful %s blocks a newer restart continuation when completion timestamps tie",
1110 (stopType) =>
1111 Effect.gen(function* () {
1112 const orchestrator = yield* Orchestrator.OrchestratorV2;
1113 const projections = yield* ProjectionStore.ProjectionStoreV2;
1114 const sql = yield* SqlClient.SqlClient;
1115 const threadId = ThreadId.make("thread:stop-resumed-queue");
1116 yield* createWatchingThread(threadId, 12);
1117 yield* send(threadId, "first", "start_immediately");
1118 yield* send(threadId, "queued", "queue_after_active");
1119 const now = yield* DateTime.now;
1120 const [first, queued] = (yield* orchestrator.getThreadProjection(threadId)).runs;
1121 assert.isDefined(first);
1122 assert.isDefined(queued);
1123 for (const run of [first!, queued!]) {
1124 yield* projections.apply({
1125 id: EventId.make(`event:stop-resumed-queue:hold:${run.id}`),
1126 type: "run.updated",
1127 threadId,
1128 runId: run.id,
1129 occurredAt: now,
1130 payload:
1131 run.id === first!.id
1132 ? { ...run, status: "completed", completedAt: now }
1133 : { ...run, queueHeld: true },
1134 });
1135 }
1136 yield* send(threadId, "restart source", "start_immediately");
1137 const source = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!;
1138 yield* projections.apply({
1139 id: EventId.make("event:stop-resumed-queue:restart-cancelled"),
1140 type: "run.updated",
1141 threadId,
1142 runId: source.id,
1143 occurredAt: now,
1144 payload: { ...source, status: "cancelled", startedAt: now, completedAt: now },
1145 });
1146 const at = DateTime.formatIso(now);
1147 yield* sql`
1148 INSERT INTO orchestration_v2_effect_outbox (
1149 effect_id, command_id, thread_id, effect_type, payload_json, status,
1150 attempt_count, available_at, created_at, updated_at
1151 ) VALUES (
1152 ${`effect:restart-continuation:${source.id}`}, 'command:restart', ${threadId},
1153 'provider-runtime.continue',
1154 ${encodeEffectRequest({ type: "provider-runtime.continue", sourceRunId: source.id })},
1155 'pending', 0, ${at}, ${at}, ${at}
1156 )
1157 `;
1158 yield* orchestrator.dispatch({
1159 type: "queue.resume",
1160 commandId: CommandId.make("resume-older-queue"),
1161 threadId,
1162 });
1163 assert.equal(
1164 (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1165 (run) => run.id === queued!.id,
1166 )?.status,
1167 "starting",
1168 );
1169 yield* orchestrator.dispatch(
1170 stopType === "thread.stop"
1171 ? { type: "thread.stop", commandId: CommandId.make("stop-resumed-queue"), threadId }
1172 : {
1173 type: "run.interrupt",
1174 commandId: CommandId.make("stop-resumed-queue"),
1175 threadId,
1176 runId: queued!.id,
1177 holdQueue: true,
1178 },
1179 );
1180 const stopped = yield* orchestrator.getThreadProjection(threadId);
1181 const interrupted = stopped.runs.find((run) => run.id === queued!.id)!;
1182 assert.equal(interrupted.status, "interrupted");
1183 assert.equal(DateTime.toEpochMillis(interrupted.completedAt!), DateTime.toEpochMillis(now));
1184 yield* orchestrator.dispatch({
1185 type: "message.dispatch",
1186 commandId: CommandId.make("restart-after-stop"),
1187 threadId,
1188 messageId: MessageId.make("message:restart-after-stop"),
1189 text: "Continue.",
1190 attachments: [],
1191 dispatchMode: { type: "start_immediately" },
1192 createdBy: "agent",
1193 creationSource: "server",
1194 restartContinuationOfRunId: source.id,
1195 });
1196 assert.deepEqual(yield* threadState(threadId), {
1197 runs: ["completed", "cancelled", "interrupted"],
1198 watched: [],
1199 });
1200 }).pipe(Effect.provide(layerTest.pipe(Layer.provideMerge(TestClock.layer())))),
1201 );
1202
1203 it.effect.each([
1204 { ending: "cancelled", question: "resolved" },
1205 { ending: "interrupted", question: "resolved" },
1206 { ending: "completed", question: "pending" },
1207 { ending: "failed", question: "pending" },
1208 ] as const)(
1209 "a $ending delegated task leaves its async question $question",
1210 ({ ending, question }) =>
1211 Effect.gen(function* () {
1212 const orchestrator = yield* Orchestrator.OrchestratorV2;
1213 const projections = yield* ProjectionStore.ProjectionStoreV2;
1214 const parentThreadId = ThreadId.make(`thread:stop-question-parent:${ending}`);
1215 yield* createWatchingThread(parentThreadId, 20);
1216 yield* send(parentThreadId, "first", "start_immediately");
1217 const childThreadId = yield* delegate(parentThreadId, `question task ${ending}`);
1218 const childRun = (yield* orchestrator.getThreadProjection(childThreadId)).runs[0]!;
1219 assert.equal(
1220 (yield* orchestrator.getThreadProjection(childThreadId)).nodes.find(
1221 (node) => node.id === childRun.rootNodeId,
1222 )?.runId,
1223 childRun.id,
1224 );
1225 const now = yield* DateTime.now;
1226 yield* projections.apply({
1227 id: EventId.make("event:stop-question:request"),
1228 type: "runtime-request.updated",
1229 threadId: childThreadId,
1230 runId: childRun.id,
1231 occurredAt: now,
1232 payload: {
1233 id: RuntimeRequestId.make("request:stop-question"),
1234 nodeId: childRun.rootNodeId!,
1235 providerTurnId: null,
1236 nativeRequestRef: null,
1237 kind: "user_input",
1238 status: "pending",
1239 responseCapability: { type: "message" },
1240 createdAt: now,
1241 resolvedAt: null,
1242 },
1243 });
1244 const request = Effect.map(
1245 orchestrator.getThreadProjection(childThreadId),
1246 (projection) => projection.runtimeRequests[0],
1247 );
1248
1249 yield* orchestrator.recoverDelegatedTask(childThreadId, { sourceRunId: childRun.id });
1250 assert.equal((yield* request)?.status, "pending");
1251 if (ending === "interrupted") {
1252 yield* orchestrator.dispatch({
1253 type: "thread.stop",
1254 commandId: CommandId.make("stop-question-child"),
1255 threadId: childThreadId,
1256 });
1257 } else {
1258 yield* projections.apply({
1259 id: EventId.make(`event:stop-question:${ending}`),
1260 type: "run.updated",
1261 threadId: childThreadId,
1262 runId: childRun.id,
1263 occurredAt: now,
1264 payload: { ...childRun, status: ending, startedAt: now, completedAt: now },
1265 });
1266 }
1267 yield* orchestrator.recoverDelegatedTask(childThreadId, { sourceRunId: childRun.id });
1268 assert.equal((yield* request)?.status, question);
1269 assert.equal((yield* request)?.decision, question === "resolved" ? "cancel" : undefined);
1270 }).pipe(Effect.provide(layerTest)),
1271 );
1272
1273 it.effect("a delegated task that cannot be stopped fails the walk after its siblings stop", () =>
1274 Effect.gen(function* () {
1275 const orchestrator = yield* Orchestrator.OrchestratorV2;
1276 const projections = yield* ProjectionStore.ProjectionStoreV2;
1277 const threads = yield* ThreadManagementService.ThreadManagementService;
1278 const parentThreadId = ThreadId.make("thread:stop-partial");
1279 yield* createWatchingThread(parentThreadId, 6);
1280 yield* send(parentThreadId, "work", "start_immediately");
1281 const childThreadId = yield* delegate(parentThreadId, "stoppable task");
1282 const task = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]!;
1283 yield* projections.apply({
1284 id: EventId.make("event:stop-partial:missing-task"),
1285 type: "subagent.updated",
1286 threadId: parentThreadId,
1287 runId: task.runId!,
1288 nodeId: NodeId.make("node:stop-partial:missing"),
1289 providerInstanceId: instanceId,
1290 occurredAt: yield* DateTime.now,
1291 payload: {
1292 ...task,
1293 id: NodeId.make("node:stop-partial:missing"),
1294 childThreadId: ThreadId.make("thread:stop-partial:missing"),
1295 startedAt: DateTime.subtract(task.startedAt!, { hours: 1 }),
1296 },
1297 });
1298
1299 const walked = yield* Effect.exit(
1300 threads.stopDelegatedTasks({
1301 threadId: parentThreadId,
1302 commandId: CommandId.make("stop-partial"),
1303 }),
1304 );
1305 assert.isTrue(Exit.isFailure(walked));
1306 assert.deepEqual((yield* threadState(childThreadId)).runs, ["interrupted"]);
1307 }).pipe(Effect.provide(layerTest)),
1308 );
1309
1310 it.effect("thread.stop marks a turn it cannot interrupt so a late agent watch is refused", () =>
1311 Effect.gen(function* () {
1312 const orchestrator = yield* Orchestrator.OrchestratorV2;
1313 const projections = yield* ProjectionStore.ProjectionStoreV2;
1314 const threadId = ThreadId.make("thread:stop-lost-session");
1315 yield* createWatchingThread(threadId, 11);
1316 yield* send(threadId, "work", "start_immediately");
1317 const now = yield* DateTime.now;
1318 const run = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!;
1319 yield* projections.apply({
1320 id: EventId.make("event:stop-lost-session:running"),
1321 type: "run.updated",
1322 threadId,
1323 runId: run.id,
1324 occurredAt: now,
1325 payload: { ...run, status: "running", startedAt: now },
1326 });
1327 yield* projections.apply({
1328 id: EventId.make("event:stop-lost-session:turn"),
1329 type: "provider-turn.updated",
1330 threadId,
1331 occurredAt: now,
1332 payload: {
1333 id: ProviderTurnId.make("provider-turn:stop-lost-session"),
1334 providerThreadId: run.providerThreadId!,
1335 nodeId: run.rootNodeId!,
1336 runAttemptId: run.activeAttemptId!,
1337 nativeTurnRef: null,
1338 ordinal: 1,
1339 status: "running",
1340 startedAt: now,
1341 completedAt: null,
1342 },
1343 });
1344
1345 yield* orchestrator.dispatch({
1346 type: "thread.stop",
1347 commandId: CommandId.make("stop-lost-session"),
1348 threadId,
1349 });
1350 assert.isTrue(Exit.isFailure(yield* Effect.exit(watch(threadId, 12))));
1351 assert.deepEqual(yield* threadState(threadId), { runs: ["running"], watched: [] });
1352 }).pipe(Effect.provide(layerTest)),
1353 );
1354
1355 it.effect.each(["missing session", "completed", "failed", "cancelled"] as const)(
1356 "a native child Stop only settles a confirmed %s outcome and preserves its owner and sibling",
1357 (providerState) =>
1358 Effect.gen(function* () {
1359 const orchestrator = yield* Orchestrator.OrchestratorV2;
1360 const projections = yield* ProjectionStore.ProjectionStoreV2;
1361 const sql = yield* SqlClient.SqlClient;
1362 const threadId = ThreadId.make("thread:native-child-stop");
1363 yield* createWatchingThread(threadId, 20);
1364 yield* send(threadId, "owner", "start_immediately");
1365 yield* delegate(threadId, "native-child-fixture");
1366 const before = yield* orchestrator.getThreadProjection(threadId);
1367 const run = before.runs[0]!;
1368 const now = yield* DateTime.now;
1369 const subagent = {
1370 ...before.subagents[0]!,
1371 origin: "provider_native" as const,
1372 driver: CLAUDE_PROVIDER,
1373 providerInstanceId: ProviderInstanceId.make("claudeAgent"),
1374 nativeTaskRef: {
1375 driver: CLAUDE_PROVIDER,
1376 nativeId: "claude-task",
1377 strength: "strong" as const,
1378 },
1379 status: "running" as const,
1380 startedAt: now,
1381 };
1382 yield* projections.apply({
1383 id: EventId.make("event:native-child-stop:agent"),
1384 type: "subagent.updated",
1385 threadId,
1386 runId: run.id,
1387 nodeId: subagent.id,
1388 occurredAt: now,
1389 payload: subagent,
1390 });
1391 const sibling = {
1392 ...subagent,
1393 id: NodeId.make("node:native-stop-sibling"),
1394 childThreadId: null,
1395 };
1396 yield* projections.apply({
1397 id: EventId.make("event:native-child-stop:sibling"),
1398 type: "subagent.updated",
1399 threadId,
1400 runId: run.id,
1401 nodeId: sibling.id,
1402 occurredAt: now,
1403 payload: sibling,
1404 });
1405 const item = before.turnItems.find(
1406 (item) => item.type === "subagent" && item.subagentId === subagent.id,
1407 )!;
1408 yield* projections.apply({
1409 id: EventId.make("event:native-child-stop:item"),
1410 type: "turn-item.updated",
1411 threadId,
1412 runId: run.id,
1413 occurredAt: now,
1414 payload: { ...item, status: "running", completedAt: null },
1415 });
1416 const childRoot = {
1417 ...before.nodes.find((node) => node.id === run.rootNodeId)!,
1418 id: NodeId.make("node:native-stop-child-root"),
1419 threadId: subagent.childThreadId!,
1420 runId: null,
1421 parentNodeId: null,
1422 status: "running" as const,
1423 countsForRun: false,
1424 completedAt: null,
1425 };
1426 yield* projections.apply({
1427 id: EventId.make("event:native-child-stop:child-root"),
1428 type: "node.updated",
1429 threadId: childRoot.threadId,
1430 nodeId: childRoot.id,
1431 occurredAt: now,
1432 payload: childRoot,
1433 });
1434 const childTurnId = ProviderTurnId.make("turn:native-stop-child");
1435 yield* projections.apply({
1436 id: EventId.make("event:native-child-stop:child-turn"),
1437 type: "provider-turn.updated",
1438 threadId: childRoot.threadId,
1439 nodeId: childRoot.id,
1440 occurredAt: now,
1441 payload: {
1442 id: childTurnId,
1443 providerThreadId: ProviderThreadId.make("provider-thread:native-stop-child"),
1444 nodeId: childRoot.id,
1445 runAttemptId: null,
1446 nativeTurnRef: null,
1447 ordinal: 1,
1448 status: "running",
1449 startedAt: now,
1450 completedAt: null,
1451 },
1452 });
1453 const providerThread = before.providerThreads.find(
1454 (thread) => thread.id === run.providerThreadId,
1455 )!;
1456 yield* projections.apply({
1457 id: EventId.make("event:native-child-stop:provider-thread"),
1458 type: "provider-thread.updated",
1459 threadId,
1460 occurredAt: now,
1461 payload: {
1462 ...providerThread,
1463 driver: CLAUDE_PROVIDER,
1464 providerInstanceId: ProviderInstanceId.make("claudeAgent"),
1465 providerSessionId: ProviderSessionId.make("session:native-child-stop"),
1466 },
1467 });
1468 const providerTurnId = ProviderTurnId.make("turn:native-child-stop");
1469 yield* projections.apply({
1470 id: EventId.make("event:native-child-stop:provider-turn"),
1471 type: "provider-turn.updated",
1472 threadId,
1473 occurredAt: now,
1474 payload: {
1475 id: providerTurnId,
1476 providerThreadId: providerThread.id,
1477 nodeId: run.rootNodeId!,
1478 runAttemptId: run.activeAttemptId!,
1479 nativeTurnRef: null,
1480 ordinal: 1,
1481 status: "completed",
1482 startedAt: now,
1483 completedAt: now,
1484 },
1485 });
1486 yield* projections.apply({
1487 id: EventId.make("event:native-child-stop:owner-completed"),
1488 type: "run.updated",
1489 threadId,
1490 runId: run.id,
1491 occurredAt: now,
1492 payload: { ...run, status: "completed", startedAt: now, completedAt: now },
1493 });
1494 yield* send(threadId, "queued", "queue_after_active");
1495 const stateBefore = yield* threadState(threadId);
1496 const eventStore = yield* EventStore.EventStoreV2;
1497 const afterSequence = yield* eventStore.latestSequence({ threadId });
1498 const confirmedStatus =
1499 providerState === "completed" || providerState === "failed" || providerState === "cancelled"
1500 ? providerState
1501 : null;
1502 const confirmedOutcome =
1503 confirmedStatus === null
1504 ? undefined
1505 : { status: confirmedStatus, completedAt: now, result: "Actual provider outcome" };
1506 const commandId = CommandId.make("stop-native-child");
1507 yield* orchestrator.dispatch({
1508 type: "subagent.stop",
1509 commandId,
1510 threadId,
1511 runId: run.id,
1512 subagentId: subagent.id,
1513 });
1514 assert.deepEqual(yield* threadState(threadId), stateBefore);
1515 const effects = yield* sql<{ payload_json: string }>`
1516 SELECT payload_json FROM orchestration_v2_effect_outbox WHERE command_id = ${commandId}
1517 `;
1518 assert.equal(effects.length, 1);
1519 assert.deepEqual(JSON.parse(effects[0]!.payload_json), {
1520 type: "provider-turn.interrupt",
1521 providerSessionId: "session:native-child-stop",
1522 providerThreadId: providerThread.id,
1523 providerTurnId,
1524 subagent: { id: subagent.id, nativeTaskId: "claude-task" },
1525 });
1526 assert.equal(
1527 (yield* orchestrator.getThreadProjection(threadId)).subagents[0]?.status,
1528 "running",
1529 );
1530 const unrelated = yield* Effect.exit(
1531 orchestrator.dispatch({
1532 type: "subagent.stop",
1533 commandId: CommandId.make("stop-unknown-child"),
1534 threadId,
1535 runId: run.id,
1536 subagentId: NodeId.make("unknown-child"),
1537 }),
1538 );
1539 assert.isTrue(Exit.isFailure(unrelated));
1540 yield* TestClock.adjust("1 second");
1541 yield* sql`DELETE FROM orchestration_v2_effect_outbox WHERE command_id != ${commandId}`;
1542 if (providerState === "missing session") {
1543 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1544 yield* worker.drain();
1545 } else {
1546 yield* orchestrator.dispatch({
1547 type: "thread.background-work.settle",
1548 commandId: CommandId.make("repair-native-child"),
1549 threadId,
1550 providerThreadId: providerThread.id,
1551 providerTurnId,
1552 subagent: { id: subagent.id, stoppedAt: now, outcome: confirmedOutcome! },
1553 });
1554 }
1555 const settled = yield* orchestrator.getThreadProjection(threadId);
1556 const cancelled = settled.subagents.find((task) => task.id === subagent.id)!;
1557 if (confirmedOutcome === undefined) {
1558 const events = yield* eventStore.read({ threadId, afterSequence }).pipe(Stream.runCollect);
1559 assert.isFalse(
1560 events.some(
1561 ({ event }) =>
1562 (event.type === "subagent.updated" ||
1563 event.type === "node.updated" ||
1564 event.type === "turn-item.updated") &&
1565 ["completed", "failed", "cancelled", "interrupted"].includes(event.payload.status),
1566 ),
1567 );
1568 assert.equal(cancelled.status, "running");
1569 assert.equal(
1570 (yield* orchestrator.getThreadProjection(childRoot.threadId)).nodes.find(
1571 (node) => node.id === childRoot.id,
1572 )?.status,
1573 "running",
1574 );
1575 assert.deepEqual(yield* threadState(threadId), stateBefore);
1576 return;
1577 }
1578
1579 const repaired = yield* eventStore.read({ threadId, afterSequence }).pipe(Stream.runCollect);
1580 const terminal = repaired.flatMap(({ event }) =>
1581 (event.type === "subagent.updated" ||
1582 event.type === "node.updated" ||
1583 event.type === "turn-item.updated") &&
1584 event.payload.status === confirmedStatus
1585 ? [event]
1586 : [],
1587 );
1588 assert.deepEqual(terminal.map((event) => event.type).sort(), [
1589 "node.updated",
1590 "subagent.updated",
1591 "turn-item.updated",
1592 ]);
1593 for (const event of terminal) {
1594 assert.equal(event.nodeId, subagent.id);
1595 assert.equal(
1596 DateTime.toEpochMillis(event.payload.completedAt!),
1597 DateTime.toEpochMillis(confirmedOutcome!.completedAt),
1598 );
1599 if (event.type === "subagent.updated" || event.type === "turn-item.updated") {
1600 assert.isDefined(event.payload.updatedAt);
1601 }
1602 }
1603
1604 assert.equal(cancelled.status, confirmedStatus);
1605 assert.isNotNull(cancelled.completedAt);
1606 assert.equal(
1607 DateTime.toEpochMillis(cancelled.completedAt!),
1608 DateTime.toEpochMillis(confirmedOutcome!.completedAt),
1609 );
1610 assert.equal(cancelled.result, confirmedOutcome!.result);
1611 assert.equal(
1612 settled.turnItems.find((candidate) => candidate.id === item.id)?.status,
1613 confirmedStatus,
1614 );
1615 const child = yield* orchestrator.getThreadProjection(childRoot.threadId);
1616 assert.equal(child.nodes.find((node) => node.id === childRoot.id)?.status, confirmedStatus);
1617 assert.equal(
1618 child.providerTurns.find((turn) => turn.id === childTurnId)?.status,
1619 confirmedStatus,
1620 );
1621 assert.isNotNull(child.nodes.find((node) => node.id === childRoot.id)?.completedAt);
1622 assert.equal(settled.subagents.find((task) => task.id === sibling.id)?.status, "running");
1623 assert.deepEqual(yield* threadState(threadId), stateBefore);
1624 yield* TestClock.adjust("1 second");
1625 const resumedAt = yield* DateTime.now;
1626 yield* projections.apply({
1627 id: EventId.make("event:native-child-stop:resume"),
1628 type: "subagent.updated",
1629 threadId,
1630 runId: run.id,
1631 nodeId: subagent.id,
1632 occurredAt: resumedAt,
1633 payload: { ...cancelled, status: "running", startedAt: resumedAt, completedAt: null },
1634 });
1635 yield* orchestrator.dispatch({
1636 type: "thread.background-work.settle",
1637 commandId: CommandId.make("old-stop-after-resume"),
1638 threadId,
1639 providerThreadId: providerThread.id,
1640 providerTurnId,
1641 subagent: { id: subagent.id, stoppedAt: now, outcome: confirmedOutcome },
1642 });
1643 assert.equal(
1644 (yield* orchestrator.getThreadProjection(threadId)).subagents.find(
1645 (task) => task.id === subagent.id,
1646 )?.status,
1647 "running",
1648 );
1649 }).pipe(Effect.provide(layerTest)),
1650 );
1651
1652 it.effect("an unanswered Stop is flagged until Force stop ends the run with its own reason", () =>
1653 Effect.gen(function* () {
1654 const orchestrator = yield* Orchestrator.OrchestratorV2;
1655 const projections = yield* ProjectionStore.ProjectionStoreV2;
1656 const threadId = ThreadId.make("thread:force-stop");
1657 yield* createWatchingThread(threadId, 6);
1658 yield* send(threadId, "work", "start_immediately");
1659 const now = yield* DateTime.now;
1660 const started = yield* orchestrator.getThreadProjection(threadId);
1661 const run = started.runs[0]!;
1662 const providerThreadId = run.providerThreadId!;
1663 const providerTurnId = ProviderTurnId.make("provider-turn:force-stop");
1664 const apply = (event: Omit<OrchestrationV2DomainEvent, "id" | "threadId">) =>
1665 projections.apply({
1666 ...event,
1667 id: EventId.make(`event:force-stop:${event.type}`),
1668 threadId,
1669 } as OrchestrationV2DomainEvent);
1670 const attempt = started.attempts[0]!;
1671 const attemptId = attempt.id;
1672 yield* apply({
1673 type: "run.updated",
1674 runId: run.id,
1675 occurredAt: now,
1676 payload: { ...run, status: "running", startedAt: now, activeAttemptId: attemptId },
1677 });
1678 yield* apply({
1679 type: "run-attempt.updated",
1680 occurredAt: now,
1681 payload: { ...attempt, status: "running", startedAt: now },
1682 });
1683 yield* apply({
1684 type: "provider-turn.updated",
1685 occurredAt: now,
1686 payload: {
1687 id: providerTurnId,
1688 providerThreadId,
1689 nodeId: run.rootNodeId!,
1690 runAttemptId: attemptId,
1691 nativeTurnRef: null,
1692 ordinal: 1,
1693 status: "running",
1694 startedAt: now,
1695 completedAt: null,
1696 },
1697 });
1698 yield* apply({
1699 type: "turn-item.updated",
1700 runId: run.id,
1701 occurredAt: now,
1702 payload: {
1703 id: TurnItemId.make("turn-item:force-stop:interrupt-request"),
1704 threadId,
1705 runId: run.id,
1706 nodeId: run.rootNodeId!,
1707 providerThreadId,
1708 providerTurnId,
1709 nativeItemRef: null,
1710 parentItemId: null,
1711 ordinal: 1_000,
1712 status: "completed",
1713 title: "Interrupt requested",
1714 startedAt: now,
1715 completedAt: now,
1716 updatedAt: now,
1717 type: "run_interrupt_request",
1718 message: "Stopped by user",
1719 },
1720 });
1721 const target = { threadId, providerThreadId, providerTurnId };
1722 const request = Effect.map(orchestrator.getThreadProjection(threadId), (projection) =>
1723 projection.turnItems.find((item) => item.type === "run_interrupt_request"),
1724 );
1725
1726 yield* orchestrator.dispatch({
1727 ...target,
1728 type: "thread.interrupt.unresponsive",
1729 commandId: CommandId.make("force-stop:unresponsive"),
1730 });
1731 const flagged = yield* request;
1732 assert.isTrue(
1733 flagged?.type === "run_interrupt_request" && flagged.unresponsiveAt !== undefined,
1734 );
1735 assert.deepEqual((yield* threadState(threadId)).runs, ["running"]);
1736
1737 yield* orchestrator.dispatch({
1738 ...target,
1739 type: "thread.background-work.settle",
1740 commandId: CommandId.make("force-stop:settle"),
1741 reason: "Force stopped by user",
1742 });
1743 const settled = yield* orchestrator.getThreadProjection(threadId);
1744 assert.deepEqual(
1745 settled.runs.map((candidate) => candidate.status),
1746 ["interrupted"],
1747 );
1748 const result = settled.turnItems.find((item) => item.type === "run_interrupt_result");
1749 assert.equal(
1750 result?.type === "run_interrupt_result" ? result.message : undefined,
1751 "Force stopped by user",
1752 );
1753
1754 // A provider confirming late changes nothing once the run has ended.
1755 const late = yield* orchestrator.dispatch({
1756 ...target,
1757 type: "thread.interrupt.unresponsive",
1758 commandId: CommandId.make("force-stop:late-unresponsive"),
1759 });
1760 assert.lengthOf(late.storedEvents, 0);
1761 }).pipe(Effect.provide(layerTest)),
1762 );
1763
1764 it.effect.each(["Stop", "Force stop"] as const)(
1765 "%s ends a running run whose provider turn already completed",
1766 (action) =>
1767 Effect.gen(function* () {
1768 const orchestrator = yield* Orchestrator.OrchestratorV2;
1769 const projections = yield* ProjectionStore.ProjectionStoreV2;
1770 const threadId = ThreadId.make("thread:stop-completed-turn");
1771 yield* createWatchingThread(threadId, 7);
1772 yield* send(threadId, "go", "start_immediately");
1773 const now = yield* DateTime.now;
1774 const started = yield* orchestrator.getThreadProjection(threadId);
1775 const run = started.runs[0]!;
1776 const attempt = started.attempts[0]!;
1777 const providerThreadId = run.providerThreadId!;
1778 const providerTurnId = ProviderTurnId.make("provider-turn:stop-completed-turn");
1779 const apply = (event: Omit<OrchestrationV2DomainEvent, "id" | "threadId">) =>
1780 projections.apply({
1781 ...event,
1782 id: EventId.make(`event:stop-completed-turn:${event.type}`),
1783 threadId,
1784 } as OrchestrationV2DomainEvent);
1785 yield* apply({
1786 type: "run.updated",
1787 runId: run.id,
1788 occurredAt: now,
1789 payload: { ...run, status: "running", startedAt: now, activeAttemptId: attempt.id },
1790 });
1791 yield* apply({
1792 type: "run-attempt.updated",
1793 occurredAt: now,
1794 payload: { ...attempt, status: "completed", startedAt: now, completedAt: now },
1795 });
1796 yield* apply({
1797 type: "provider-turn.updated",
1798 occurredAt: now,
1799 payload: {
1800 id: providerTurnId,
1801 providerThreadId,
1802 nodeId: run.rootNodeId!,
1803 runAttemptId: attempt.id,
1804 nativeTurnRef: null,
1805 ordinal: 1,
1806 status: "completed",
1807 startedAt: now,
1808 completedAt: now,
1809 },
1810 });
1811 // The provider session was already released.
1812 yield* apply({
1813 type: "provider-thread.updated",
1814 occurredAt: now,
1815 payload: {
1816 ...started.providerThreads.find((thread) => thread.id === providerThreadId)!,
1817 providerSessionId: ProviderSessionId.make("session:gone"),
1818 },
1819 });
1820 yield* send(threadId, "follow", "queue_after_active");
1821
1822 if (action === "Stop") {
1823 yield* orchestrator.dispatch({
1824 type: "thread.stop",
1825 commandId: CommandId.make("stop-completed-turn"),
1826 threadId,
1827 });
1828 } else {
1829 const threads = yield* ThreadManagementService.ThreadManagementService;
1830 yield* threads.forceStopThread({ threadId });
1831 }
1832 const stopped = yield* orchestrator.getThreadProjection(threadId);
1833 assert.notEqual(stopped.runs[0]?.status, "running", "the stopped run is still running");
1834 assert.deepInclude(stopped.attempts[0], { status: "completed", completedAt: now });
1835 assert.isTrue(
1836 stopped.turnItems.some((item) => item.type === "run_interrupt_result"),
1837 "the stopped run has no interrupt result",
1838 );
1839 }).pipe(Effect.provide(layerTest)),
1840 );
1841
1842 // A finished delegated child whose native subagent the provider never reported ending,
1843 // with no open turn item, and a nested native subagent on that agent's own thread.
1844 it.effect(
1845 "Stop settles a delegated row whose finished child still records a running native subagent",
1846 () =>
1847 Effect.gen(function* () {
1848 const orchestrator = yield* Orchestrator.OrchestratorV2;
1849 const projections = yield* ProjectionStore.ProjectionStoreV2;
1850 const parentThreadId = ThreadId.make("thread:held-row");
1851 yield* createWatchingThread(parentThreadId, 30);
1852 yield* send(parentThreadId, "parent", "start_immediately");
1853 const childThreadId = yield* delegate(parentThreadId, "held child");
1854 const child = yield* orchestrator.getThreadProjection(childThreadId);
1855 const childRun = child.runs[0]!;
1856 const task = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]!;
1857 const now = yield* DateTime.now;
1858 const nativeThreadId = ThreadId.make("thread:held-row-native");
1859 const nestedThreadId = ThreadId.make("thread:held-row-nested");
1860 yield* createWatchingThread(nativeThreadId, 31);
1861 yield* createWatchingThread(nestedThreadId, 32);
1862 const nativeProviderThreadId = ProviderThreadId.make("provider-thread:held-row-native");
1863 const runningRoot = (threadId: ThreadId, id: NodeId) => ({
1864 ...child.nodes.find((node) => node.id === childRun.rootNodeId)!,
1865 id,
1866 rootNodeId: id,
1867 threadId,
1868 runId: null,
1869 parentNodeId: null,
1870 status: "running" as const,
1871 countsForRun: false,
1872 completedAt: null,
1873 });
1874 const nativeRoot = runningRoot(nativeThreadId, NodeId.make("node:held-row-native-root"));
1875 const nestedRoot = runningRoot(nestedThreadId, NodeId.make("node:held-row-nested-root"));
1876 for (const root of [nativeRoot, nestedRoot]) {
1877 yield* projections.apply({
1878 id: EventId.make(`event:held-row:${root.id}`),
1879 type: "node.updated",
1880 threadId: root.threadId,
1881 nodeId: root.id,
1882 occurredAt: now,
1883 payload: root,
1884 });
1885 }
1886 yield* projections.apply({
1887 id: EventId.make("event:held-row:native-turn"),
1888 type: "provider-turn.updated",
1889 threadId: nativeThreadId,
1890 nodeId: nativeRoot.id,
1891 occurredAt: now,
1892 payload: {
1893 id: ProviderTurnId.make("turn:held-row-native"),
1894 providerThreadId: nativeProviderThreadId,
1895 nodeId: nativeRoot.id,
1896 runAttemptId: null,
1897 nativeTurnRef: null,
1898 ordinal: 1,
1899 status: "running",
1900 startedAt: now,
1901 completedAt: null,
1902 },
1903 });
1904 const native = {
1905 ...task,
1906 id: NodeId.make("node:held-row-native"),
1907 threadId: childThreadId,
1908 runId: childRun.id,
1909 parentNodeId: childRun.rootNodeId!,
1910 origin: "provider_native" as const,
1911 providerThreadId: nativeProviderThreadId,
1912 childThreadId: nativeThreadId,
1913 status: "running" as const,
1914 completionDelivery: undefined,
1915 };
1916 yield* projections.apply({
1917 id: EventId.make("event:held-row:native"),
1918 type: "subagent.updated",
1919 threadId: childThreadId,
1920 runId: childRun.id,
1921 occurredAt: now,
1922 payload: native,
1923 });
1924 yield* projections.apply({
1925 id: EventId.make("event:held-row:nested"),
1926 type: "subagent.updated",
1927 threadId: nativeThreadId,
1928 occurredAt: now,
1929 payload: {
1930 ...native,
1931 id: NodeId.make("node:held-row-nested"),
1932 threadId: nativeThreadId,
1933 runId: null,
1934 parentNodeId: nativeRoot.id,
1935 childThreadId: nestedThreadId,
1936 },
1937 });
1938 yield* projections.apply({
1939 id: EventId.make("event:held-row:child-completed"),
1940 type: "run.updated",
1941 threadId: childThreadId,
1942 runId: childRun.id,
1943 occurredAt: now,
1944 payload: { ...childRun, status: "completed", startedAt: now, completedAt: now },
1945 });
1946 yield* orchestrator.recoverDelegatedTasks;
1947 assert.equal(
1948 (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]?.status,
1949 "running",
1950 );
1951 const statuses = Effect.gen(function* () {
1952 const nativeAfter = yield* orchestrator.getThreadProjection(nativeThreadId);
1953 const nestedAfter = yield* orchestrator.getThreadProjection(nestedThreadId);
1954 return {
1955 row: (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]?.status,
1956 native: (yield* orchestrator.getThreadProjection(childThreadId)).subagents.find(
1957 (subagent) => subagent.id === native.id,
1958 )?.status,
1959 nativeRoot: nativeAfter.nodes.find((node) => node.id === nativeRoot.id)?.status,
1960 nested: nativeAfter.subagents[0]?.status,
1961 nestedRoot: nestedAfter.nodes.find((node) => node.id === nestedRoot.id)?.status,
1962 };
1963 });
1964
1965 // The row's Stop: the parent's task, then the child thread.
1966 yield* orchestrator.dispatch({
1967 type: "subagent.stop",
1968 commandId: CommandId.make("held-row:stop-task"),
1969 threadId: parentThreadId,
1970 runId: task.runId!,
1971 subagentId: task.id,
1972 });
1973 yield* orchestrator.dispatch({
1974 type: "thread.stop",
1975 commandId: CommandId.make("held-row:stop-child"),
1976 threadId: childThreadId,
1977 });
1978 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1979 yield* worker.drain();
1980 assert.deepEqual(yield* statuses, {
1981 row: "completed",
1982 native: "interrupted",
1983 nativeRoot: "interrupted",
1984 nested: "interrupted",
1985 nestedRoot: "interrupted",
1986 });
1987 }).pipe(Effect.provide(layerTest)),
1988 );
1989
1990 it.effect.each([
1991 { followUp: "started its turn", startedBeforeRelease: true },
1992 { followUp: "was promoted", startedBeforeRelease: false },
1993 { followUp: "switched provider instance", startedBeforeRelease: true, switchesInstance: true },
1994 { followUp: "has fresh context", startedBeforeRelease: true, freshContext: true },
1995 ])(
1996 "Force stop keeps only the session of a queued follow-up that $followUp",
1997 ({ startedBeforeRelease, switchesInstance = false, freshContext = false }) =>
1998 Effect.gen(function* () {
1999 const cwd = yield* checkpointWorkspace("thread-stop-force-stop-follow-up");
2000 const driver = adapter.driver;
2001 const otherInstanceId = ProviderInstanceId.make("codex-other");
2002 const eventQueues = new Map<
2003 ProviderSessionId,
2004 Queue.Queue<ProviderAdapter.ProviderAdapterV2Event>
2005 >();
2006 const started: Array<ProviderAdapter.ProviderAdapterV2TurnInput> = [];
2007 const interrupted: Array<ProviderAdapter.ProviderAdapterV2InterruptInput> = [];
2008 let opened = 0;
2009 // Fresh context on a one-thread-per-session provider puts the follow-up on its own session.
2010 // The session itself reports shared threads so fresh-context cleanup leaves the stopped
2011 // run's session to Force stop, as pending background work would.
2012 const adapterCapabilities = freshContext
2013 ? {
2014 ...CodexProviderCapabilitiesV2,
2015 sessions: {
2016 ...CodexProviderCapabilitiesV2.sessions,
2017 supportsMultipleProviderThreadsPerSession: false,
2018 },
2019 }
2020 : CodexProviderCapabilitiesV2;
2021 const liveAdapter = (
2022 instanceId: ProviderInstanceId,
2023 ): ProviderAdapter.ProviderAdapterV2["Service"] => ({
2024 ...adapter,
2025 instanceId,
2026 getCapabilities: () => Effect.succeed(adapterCapabilities),
2027 openSession: (input) =>
2028 Effect.gen(function* () {
2029 opened += 1;
2030 const now = yield* DateTime.now;
2031 const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
2032 eventQueues.set(input.providerSessionId, events);
2033 return {
2034 instanceId,
2035 driver,
2036 providerSessionId: input.providerSessionId,
2037 providerSession: {
2038 id: input.providerSessionId,
2039 driver,
2040 providerInstanceId: instanceId,
2041 status: "ready",
2042 cwd,
2043 model: modelSelection.model,
2044 capabilities: CodexProviderCapabilitiesV2,
2045 createdAt: now,
2046 updatedAt: now,
2047 lastError: null,
2048 },
2049 events: Stream.fromQueue(events),
2050 ensureThread: ({ threadId }) =>
2051 Effect.succeed({
2052 id: ProviderThreadId.make(
2053 `provider-thread:${input.providerSessionId}:${threadId}`,
2054 ),
2055 driver,
2056 providerInstanceId: instanceId,
2057 providerSessionId: input.providerSessionId,
2058 appThreadId: threadId,
2059 ownerNodeId: null,
2060 nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" },
2061 nativeConversationHeadRef: null,
2062 status: "idle",
2063 firstRunOrdinal: null,
2064 lastRunOrdinal: null,
2065 handoffIds: [],
2066 forkedFrom: null,
2067 createdAt: now,
2068 updatedAt: now,
2069 }),
2070 resumeThread: ({ providerThread }) => Effect.succeed(providerThread),
2071 startTurn: (turn) =>
2072 Effect.gen(function* () {
2073 started.push(turn);
2074 yield* Queue.offer(events, {
2075 type: "provider_turn.updated",
2076 driver,
2077 providerTurn: {
2078 id: ProviderTurnId.make(`provider-turn:${turn.attemptId}`),
2079 providerThreadId: turn.providerThread.id,
2080 nodeId: turn.rootNodeId,
2081 runAttemptId: turn.attemptId,
2082 nativeTurnRef: {
2083 driver,
2084 nativeId: `native:${turn.attemptId}`,
2085 strength: "strong",
2086 },
2087 ordinal: turn.providerTurnOrdinal,
2088 status: "running",
2089 startedAt: now,
2090 completedAt: null,
2091 },
2092 });
2093 }),
2094 steerTurn: () => Effect.die("unused"),
2095 interruptTurn: (interrupt) => Effect.sync(() => interrupted.push(interrupt)),
2096 respondToRuntimeRequest: () => Effect.die("unused"),
2097 readThreadSnapshot: () => Effect.die("unused"),
2098 rollbackThread: () => Effect.die("unused"),
2099 forkThread: () => Effect.die("unused"),
2100 };
2101 }),
2102 });
2103 // The settlement promotes the follow-up; optionally its turn starts before Force stop releases.
2104 const layerSessionsStartingFollowUpFirst = Layer.effect(
2105 ProviderSessionManager.ProviderSessionManagerV2,
2106 Effect.gen(function* () {
2107 const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
2108 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
2109 return ProviderSessionManager.ProviderSessionManagerV2.of({
2110 ...sessions,
2111 releaseUnlessBusy: (input) =>
2112 (startedBeforeRelease ? worker.drain().pipe(Effect.orDie) : Effect.void).pipe(
2113 Effect.andThen(sessions.releaseUnlessBusy(input)),
2114 ),
2115 });
2116 }),
2117 );
2118 const layerLive = ThreadManagementService.layer.pipe(
2119 Layer.provide(layerSessionsStartingFollowUpFirst),
2120 Layer.provideMerge(
2121 Layer.mergeAll(
2122 layerDatabase,
2123 ProjectionStore.layer.pipe(Layer.provide(layerDatabase)),
2124 EventStore.layer.pipe(Layer.provide(layerDatabase)),
2125 EffectOutbox.layer.pipe(Layer.provide(layerDatabase)),
2126 ProviderReplayHarness.layerWithRegistry(
2127 { name: "thread-stop-force-stop-follow-up" },
2128 ProviderAdapterRegistry.layerFromAdapters([
2129 liveAdapter(instanceId),
2130 liveAdapter(otherInstanceId),
2131 ]),
2132 { databaseLayer: layerDatabase, runEffectWorker: false },
2133 ),
2134 ),
2135 ),
2136 Layer.provideMerge(ServerSettings.layerTest()),
2137 Layer.provideMerge(CommandReceiptStore.layer.pipe(Layer.provide(layerDatabase))),
2138 );
2139
2140 yield* Effect.gen(function* () {
2141 const orchestrator = yield* Orchestrator.OrchestratorV2;
2142 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
2143 const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
2144 const threads = yield* ThreadManagementService.ThreadManagementService;
2145 const threadId = ThreadId.make("thread:force-stop-follow-up");
2146 const runningTurn = orchestrator.streamDomainEvents.pipe(
2147 Stream.filter(
2148 (event) => event.type === "provider-turn.updated" && event.payload.status === "running",
2149 ),
2150 Stream.take(1),
2151 Stream.runDrain,
2152 Effect.forkScoped,
2153 );
2154 yield* orchestrator.dispatch({
2155 type: "thread.create",
2156 commandId: CommandId.make("create:force-stop-follow-up"),
2157 threadId,
2158 projectId: ProjectId.make("project:thread-stop"),
2159 title: "Force stop follow-up",
2160 modelSelection,
2161 runtimeMode: "full-access",
2162 interactionMode: "default",
2163 branch: null,
2164 worktreePath: cwd,
2165 createdBy: "user",
2166 creationSource: "web",
2167 });
2168 const firstRunning = yield* runningTurn;
2169 yield* send(threadId, "first", "start_immediately");
2170 yield* worker.drain();
2171 yield* Fiber.join(firstRunning);
2172 yield* orchestrator.dispatch({
2173 type: "message.dispatch",
2174 commandId: CommandId.make("send:force-stop-follow-up"),
2175 threadId,
2176 messageId: MessageId.make("message:force-stop-follow-up"),
2177 text: "follow-up",
2178 attachments: [],
2179 dispatchMode: { type: "queue_after_active" },
2180 ...(switchesInstance
2181 ? { modelSelection: { ...modelSelection, instanceId: otherInstanceId } }
2182 : {}),
2183 ...(freshContext ? { freshContext } : {}),
2184 createdBy: "user",
2185 creationSource: "web",
2186 });
2187
2188 const result = yield* threads.forceStopThread({ threadId });
2189 yield* worker.drain();
2190
2191 assert.deepEqual(
2192 started.map((turn) => turn.runOrdinal),
2193 [1, 2],
2194 "the queued follow-up did not start its turn",
2195 );
2196 assert.equal(
2197 opened,
2198 switchesInstance || freshContext ? 2 : 1,
2199 "Force stop tore down the session the follow-up was promoted onto",
2200 );
2201 const [stopped, followUp] = [started[0]!, started[1]!];
2202 const providerSessionId = followUp.providerThread.providerSessionId!;
2203 assert.equal(
2204 yield* sessions.isLive(stopped.providerThread.providerSessionId!),
2205 !switchesInstance && !freshContext,
2206 "Force stop kept the stopped run's session for a follow-up on another session",
2207 );
2208 assert.isFalse(
2209 interrupted.some((interrupt) =>
2210 String(interrupt.providerTurnId).includes(followUp.attemptId),
2211 ),
2212 "Force stop of the first run interrupted the follow-up's turn",
2213 );
2214 assert.isTrue(
2215 yield* sessions.isLive(providerSessionId),
2216 "Force stop of the first run released the session the follow-up's turn is running on",
2217 );
2218 assert.deepEqual(
2219 result,
2220 { outcome: "released", sharedWithThreadIds: [] },
2221 "Force stop detached the thread the follow-up runs on as if its session were shared",
2222 );
2223
2224 // The first run's provider never confirms the stop; the follow-up's turn ends normally.
2225 yield* Queue.offer(eventQueues.get(providerSessionId)!, {
2226 type: "turn.terminal",
2227 driver,
2228 providerThreadId: followUp.providerThread.id,
2229 providerTurnId: ProviderTurnId.make(`provider-turn:${followUp.attemptId}`),
2230 runOrdinal: followUp.runOrdinal,
2231 status: "completed",
2232 failure: null,
2233 threadDisposition: "reusable",
2234 });
2235 yield* worker.drain();
2236 yield* TestClock.adjust("31 minutes");
2237 assert.isFalse(
2238 yield* sessions.isLive(providerSessionId),
2239 "the stopped run's unconfirmed turn kept the session from going idle",
2240 );
2241 }).pipe(Effect.provide(layerLive));
2242 }),
2243 );