The best way to control Claude Code, Codex and more from any computer or phone

main
ts 1,083 lines 42.9 KB
Raw
1 import { expect, it } from "@effect/vitest";
2 import {
3 CommandId,
4 EnvironmentId,
5 MessageId,
6 type OrchestrationV2Command,
7 type OrchestrationV2Run,
8 type OrchestrationV2ProviderThread,
9 ProviderThreadId,
10 ProviderDriverKind,
11 ProviderSessionId,
12 type OrchestrationV2StoredEvent,
13 type OrchestrationV2ThreadProjection,
14 ProjectId,
15 ProviderInstanceId,
16 RunId,
17 ThreadId,
18 } from "@t3tools/contracts";
19 import * as Deferred from "effect/Deferred";
20 import * as DateTime from "effect/DateTime";
21 import * as Duration from "effect/Duration";
22 import * as Option from "effect/Option";
23 import * as Effect from "effect/Effect";
24 import * as Fiber from "effect/Fiber";
25 import * as Layer from "effect/Layer";
26 import * as Queue from "effect/Queue";
27 import * as Stream from "effect/Stream";
28 import * as TestClock from "effect/testing/TestClock";
29
30 import { AgentScope } from "@t3tools/shared/AgentScope";
31
32 import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
33 import * as ServerSettings from "../serverSettings.ts";
34 import * as SqlitePersistence from "../persistence/Sqlite.ts";
35 import * as CommandReceiptStore from "./CommandReceiptStore.ts";
36 import * as Orchestrator from "./Orchestrator.ts";
37 import * as ProviderSessionManager from "./ProviderSessionManager.ts";
38 import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
39 import * as ThreadManagementService from "./ThreadManagementService.ts";
40 import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
41 import * as LegacyV1ThreadImporter from "./legacy/LegacyV1ThreadImporter.ts";
42 import {
43 ThreadWorktreeRestore,
44 WorktreeRecreateError,
45 } from "../workspace/restoreMissingWorktree.ts";
46
47 it("stamps authoritative provenance on commands that create threads or messages", () => {
48 const command: OrchestrationV2Command = {
49 type: "thread.create",
50 createdBy: "agent",
51 creationSource: "mcp",
52 commandId: CommandId.make("command:thread-management:create"),
53 threadId: ThreadId.make("thread:thread-management:create"),
54 projectId: ProjectId.make("project:thread-management"),
55 title: "Thread management",
56 modelSelection: {
57 instanceId: ProviderInstanceId.make("codex"),
58 model: "gpt-5-codex",
59 },
60 runtimeMode: "full-access",
61 interactionMode: "default",
62 branch: null,
63 worktreePath: null,
64 // A client cannot claim a linked environment's origin, or a parent there.
65 linkOrigin: { sessionId: "forged", label: "Somewhere else" },
66 delegatedFrom: {
67 environmentId: EnvironmentId.make("environment:elsewhere"),
68 threadId: ThreadId.make("thread:elsewhere"),
69 title: "Somewhere else",
70 },
71 // A client's message goes through intake; only the server folds one into the creation.
72 initialMessage: {
73 messageId: MessageId.make("message:thread-management:forged"),
74 text: "Forged",
75 attachments: [],
76 },
77 };
78
79 const stamped = ThreadManagementService.withCreationProvenance(command, {
80 createdBy: "user",
81 creationSource: "web",
82 });
83 expect(stamped).toMatchObject({
84 createdBy: "user",
85 creationSource: "web",
86 });
87 expect(stamped).not.toHaveProperty("linkOrigin");
88 expect(stamped).not.toHaveProperty("delegatedFrom");
89 expect(stamped).not.toHaveProperty("initialMessage");
90 });
91
92 const layerReceiptsTest = CommandReceiptStore.layer.pipe(
93 Layer.provide(SqlitePersistence.layerMemory),
94 );
95
96 const layerDispatchChokepointTest =
97 (settingsLayer: ReturnType<typeof ServerSettings.layerTest>) => (dispatched: Array<string>) =>
98 ThreadManagementService.layer.pipe(
99 Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
100 Layer.provide(
101 Layer.mock(Orchestrator.OrchestratorV2)({
102 getThreadRecords: () =>
103 Effect.succeed({
104 thread: {
105 id: ThreadId.make("thread:thread-management:chokepoint"),
106 projectId: ProjectId.make("project:thread-management:chokepoint"),
107 deletedAt: null,
108 modelSelection: {
109 instanceId: ProviderInstanceId.make("codex_work"),
110 model: "test",
111 },
112 },
113 } as unknown as OrchestrationV2ThreadProjection),
114 dispatch: (command) =>
115 Effect.sync(() => {
116 dispatched.push(command.type);
117 return { sequence: 1, storedEvents: [] } as never;
118 }),
119 }),
120 ),
121 Layer.provide(settingsLayer),
122 Layer.provideMerge(layerReceiptsTest),
123 );
124
125 it.effect(
126 "the dispatch chokepoint refuses a continuation message.dispatch on a project-disabled instance before reaching the orchestrator",
127 () => {
128 const projectId = ProjectId.make("project:thread-management:chokepoint");
129 const threadId = ThreadId.make("thread:thread-management:chokepoint");
130 const instanceId = ProviderInstanceId.make("codex_work");
131 const commandId = CommandId.make("command:thread-management:chokepoint-disabled");
132 const dispatched: Array<string> = [];
133 const layerTest = layerDispatchChokepointTest(
134 ServerSettings.layerTest({
135 providerInstances: { [instanceId]: { driver: "codex", enabled: true } },
136 projectSettingsOverrides: {
137 [projectId]: { providerInstances: { [instanceId]: { enabled: false } } },
138 },
139 }),
140 )(dispatched);
141
142 return Effect.gen(function* () {
143 const threads = yield* ThreadManagementService.ThreadManagementService;
144 const error = yield* Effect.flip(
145 threads.dispatch({
146 type: "message.dispatch",
147 commandId,
148 threadId,
149 messageId: MessageId.make("message:thread-management:chokepoint-disabled"),
150 text: "Background task completed.",
151 attachments: [],
152 dispatchMode: { type: "queue_after_active" },
153 createdBy: "agent",
154 creationSource: "server",
155 }),
156 );
157
158 expect(error).toBeInstanceOf(
159 ThreadManagementService.ThreadManagementProviderInstanceDisabledError,
160 );
161 expect(error).toMatchObject({ projectId, instanceId });
162 expect(dispatched).toEqual([]);
163 }).pipe(Effect.provide(layerTest));
164 },
165 );
166
167 it.effect(
168 "a replay of an already-accepted command id dispatches despite the instance now being disabled",
169 () => {
170 const projectId = ProjectId.make("project:thread-management:chokepoint");
171 const threadId = ThreadId.make("thread:thread-management:chokepoint");
172 const instanceId = ProviderInstanceId.make("codex_work");
173 const commandId = CommandId.make("command:thread-management:chokepoint-replay");
174 const dispatched: Array<string> = [];
175 const layerTest = layerDispatchChokepointTest(
176 ServerSettings.layerTest({
177 providerInstances: { [instanceId]: { driver: "codex", enabled: true } },
178 projectSettingsOverrides: {
179 [projectId]: { providerInstances: { [instanceId]: { enabled: false } } },
180 },
181 }),
182 )(dispatched);
183
184 return Effect.gen(function* () {
185 const receipts = yield* CommandReceiptStore.CommandReceiptStoreV2;
186 yield* receipts.insertIfAbsent({
187 commandId,
188 threadId,
189 commandType: "message.dispatch",
190 acceptedAt: yield* DateTime.now,
191 resultSequence: 1,
192 status: "accepted",
193 error: null,
194 });
195
196 const threads = yield* ThreadManagementService.ThreadManagementService;
197 yield* threads.dispatch({
198 type: "message.dispatch",
199 commandId,
200 threadId,
201 messageId: MessageId.make("message:thread-management:chokepoint-replay"),
202 text: "Background task completed.",
203 attachments: [],
204 dispatchMode: { type: "queue_after_active" },
205 createdBy: "agent",
206 creationSource: "server",
207 });
208
209 expect(dispatched).toEqual(["message.dispatch"]);
210 }).pipe(Effect.provide(layerTest));
211 },
212 );
213
214 it.effect.each([
215 { label: "fails a send that starts a turn", running: false },
216 { label: "still steers a running turn", running: true },
217 ])("sendToThread with a missing worktree that cannot be recreated $label", ({ running }) => {
218 const projectId = ProjectId.make("project:thread-management:worktree-gone");
219 const threadId = ThreadId.make("thread:thread-management:worktree-gone");
220 const dispatched: Array<string> = [];
221 const layerTest = ThreadManagementService.layerWithLegacyImporter.pipe(
222 Layer.provide(
223 Layer.succeed(ThreadWorktreeRestore, {
224 restore: () =>
225 Effect.fail(
226 new WorktreeRecreateError({
227 worktreePath: "/worktrees/gone",
228 branch: "t3/gone",
229 cause: new Error("branch_checked_out_in_worktree"),
230 }),
231 ),
232 }),
233 ),
234 Layer.provide(
235 Layer.mock(LegacyV1ThreadImporter.LegacyV1ThreadImporter)({
236 ensureTranscript: () => Effect.succeed({ importedThreadCount: 0, importedMessageCount: 0 }),
237 }),
238 ),
239 Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
240 Layer.provide(
241 Layer.mock(Orchestrator.OrchestratorV2)({
242 getThreadRecords: () =>
243 Effect.succeed({
244 thread: {
245 id: threadId,
246 projectId,
247 deletedAt: null,
248 archivedAt: null,
249 worktreePath: "/worktrees/gone",
250 branch: "t3/gone",
251 modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "test" },
252 },
253 runs: running
254 ? [
255 {
256 id: RunId.make("run:thread-management:worktree-gone"),
257 status: "running",
258 activeAttemptId: "attempt:thread-management:worktree-gone",
259 },
260 ]
261 : [],
262 providerTurns: running
263 ? [
264 {
265 runAttemptId: "attempt:thread-management:worktree-gone",
266 status: "running",
267 },
268 ]
269 : [],
270 messages: [],
271 turnItems: [],
272 } as unknown as OrchestrationV2ThreadProjection),
273 dispatch: (command) =>
274 Effect.sync(() => {
275 dispatched.push(command.type);
276 return { sequence: 1, storedEvents: [] } as never;
277 }),
278 }),
279 ),
280 Layer.provide(ServerSettings.layerTest()),
281 Layer.provideMerge(layerReceiptsTest),
282 );
283
284 return Effect.gen(function* () {
285 const threads = yield* ThreadManagementService.ThreadManagementService;
286 const error = yield* Effect.flip(
287 threads.sendToThread({
288 projectId,
289 threadId,
290 commandId: CommandId.make("command:thread-management:worktree-gone"),
291 messageId: MessageId.make("message:thread-management:worktree-gone"),
292 text: "Reply with ok.",
293 attachments: [],
294 mode: "auto",
295 createdBy: "agent",
296 creationSource: "mcp",
297 }),
298 );
299 // The mock records no message, so a dispatched send ends in the projection read.
300 expect(error._tag).toBe(
301 running
302 ? "ThreadManagementDurableRunProjectionError"
303 : "ThreadManagementWorktreeRestoreError",
304 );
305 expect(dispatched).toEqual(running ? ["message.dispatch"] : []);
306 }).pipe(Effect.provide(layerTest));
307 });
308
309 it.effect("waitForThread follows a restart continuation instead of the cut run", () =>
310 Effect.gen(function* () {
311 const projectId = ProjectId.make("project:thread-management:wait-restart");
312 const threadId = ThreadId.make("thread:thread-management:wait-restart");
313 const runId = RunId.make("run:thread-management:wait-restart");
314 const continuationId = RunId.make("run:thread-management:wait-restart:2");
315 const subscribed = yield* Deferred.make<void>();
316 const events = yield* Queue.unbounded<OrchestrationV2StoredEvent>();
317 let runs: Array<{
318 id: RunId;
319 status: OrchestrationV2Run["status"];
320 restartContinuationOfRunId?: RunId;
321 }> = [{ id: runId, status: "cancelled" }];
322 let pending = true;
323 const stored = (sequence: number, event: object) =>
324 ({ sequence, event: { threadId, ...event } }) as unknown as OrchestrationV2StoredEvent;
325 const layerTest = ThreadManagementService.layer.pipe(
326 Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
327 Layer.provide(
328 Layer.mock(Orchestrator.OrchestratorV2)({
329 getThreadEventSequence: () => Effect.succeed(0),
330 restartContinuationPending: (run) => Effect.succeed(pending && run.id === runId),
331 awaitRestartContinuation: () => Effect.never,
332 streamStoredEventsFrom: (input) =>
333 input?.eventType === "run.updated"
334 ? Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe(
335 Stream.drain,
336 Stream.concat(Stream.fromQueue(events)),
337 )
338 : Stream.never,
339 getThreadRecords: () =>
340 Effect.sync(
341 () =>
342 ({
343 thread: { id: threadId, projectId, deletedAt: null },
344 runs,
345 }) as unknown as OrchestrationV2ThreadProjection,
346 ),
347 }),
348 ),
349 Layer.provide(ServerSettings.layerTest()),
350 Layer.provide(layerReceiptsTest),
351 );
352 const service = yield* ThreadManagementService.ThreadManagementService.pipe(
353 Effect.provide(layerTest),
354 );
355 const fiber = yield* service
356 .waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 })
357 .pipe(Effect.forkChild);
358 yield* Deferred.await(subscribed);
359 const continuation = { id: continuationId, restartContinuationOfRunId: runId };
360 runs = [...runs, { ...continuation, status: "running" }];
361 pending = false;
362 yield* Queue.offer(
363 events,
364 stored(1, { type: "run.updated", payload: { ...continuation, status: "running" } }),
365 );
366 runs = [runs[0]!, { ...continuation, status: "completed" }];
367 yield* Queue.offer(
368 events,
369 stored(2, { type: "run.updated", payload: { ...continuation, status: "completed" } }),
370 );
371 const result = yield* Fiber.join(fiber);
372
373 expect(result).toMatchObject({
374 timedOut: false,
375 run: { id: continuationId, status: "completed" },
376 });
377 }),
378 );
379
380 it.effect("waitForThread can wait past an ended run for the thread's delegated work", () =>
381 Effect.gen(function* () {
382 const projectId = ProjectId.make("project:thread-management:wait-delegated");
383 const threadId = ThreadId.make("thread:thread-management:wait-delegated");
384 const runId = RunId.make("run:thread-management:wait-delegated");
385 const wakeRunId = RunId.make("run:thread-management:wait-delegated:wake");
386 const subscribed = yield* Deferred.make<void>();
387 const events = yield* Queue.unbounded<OrchestrationV2StoredEvent>();
388 const at = (seconds: number) => DateTime.makeUnsafe(seconds * 1_000);
389 const run = (id: RunId, ordinal: number, status: OrchestrationV2Run["status"], end: number) =>
390 ({
391 id,
392 ordinal,
393 status,
394 startedAt: at(end - 1),
395 completedAt: status === "running" ? null : at(end),
396 }) as unknown as OrchestrationV2Run;
397 // The run ended, but a task it delegated still runs and will wake the thread.
398 let runs = [run(runId, 1, "completed", 1)];
399 let subagents = [{ status: "running" }];
400 const stored = (sequence: number, event: object) =>
401 ({ sequence, event: { threadId, ...event } }) as unknown as OrchestrationV2StoredEvent;
402 const layerTest = ThreadManagementService.layer.pipe(
403 Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
404 Layer.provide(
405 Layer.mock(Orchestrator.OrchestratorV2)({
406 getThreadEventSequence: () => Effect.succeed(0),
407 restartContinuationPending: () => Effect.succeed(false),
408 delegatedTaskResultPending: () => Effect.succeed(false),
409 streamStoredEventsFrom: (input) =>
410 input?.eventType === "subagent.updated"
411 ? Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe(
412 Stream.drain,
413 Stream.concat(Stream.fromQueue(events)),
414 )
415 : Stream.never,
416 getThreadRecords: () =>
417 Effect.sync(
418 () =>
419 ({
420 thread: { id: threadId, projectId, deletedAt: null },
421 runs,
422 messages: [],
423 subagents,
424 providerThreads: [],
425 }) as unknown as OrchestrationV2ThreadProjection,
426 ),
427 }),
428 ),
429 Layer.provide(ServerSettings.layerTest()),
430 Layer.provide(layerReceiptsTest),
431 );
432 const service = yield* ThreadManagementService.ThreadManagementService.pipe(
433 Effect.provide(layerTest),
434 );
435 // Without the option, the ended run is the answer, as before.
436 expect(
437 yield* service.waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 }),
438 ).toMatchObject({ timedOut: false, run: { id: runId } });
439 expect(
440 yield* service.waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 }),
441 ).not.toHaveProperty("resultRun");
442
443 const fiber = yield* service
444 .waitForThread({
445 projectId,
446 threadId,
447 runId,
448 timeoutMs: 60 * 60 * 1_000,
449 untilDelegatedWorkSettled: true,
450 })
451 .pipe(Effect.forkChild);
452 // A wait that returns early fails below instead of hanging here.
453 yield* Effect.raceFirst(Deferred.await(subscribed), Fiber.await(fiber));
454 // The task finishes; its delivery wakes the thread, whose next run answers.
455 subagents = [{ status: "completed" }];
456 runs = [...runs, run(wakeRunId, 2, "completed", 3)];
457 yield* Queue.offer(events, stored(1, { type: "subagent.updated", payload: {} }));
458 const result = yield* Fiber.join(fiber);
459
460 expect(result).toMatchObject({
461 timedOut: false,
462 run: { id: runId, status: "completed" },
463 resultRun: { id: wakeRunId, status: "completed" },
464 });
465 }),
466 );
467
468 it.effect("waitForThread settles delegated work on a thread rolled back past its only run", () =>
469 Effect.gen(function* () {
470 const projectId = ProjectId.make("project:thread-management:wait-rolled-back");
471 const threadId = ThreadId.make("thread:thread-management:wait-rolled-back");
472 const runId = RunId.make("run:thread-management:wait-rolled-back");
473 const rolledBack = {
474 id: runId,
475 ordinal: 1,
476 status: "rolled_back",
477 startedAt: DateTime.makeUnsafe(0),
478 completedAt: DateTime.makeUnsafe(1_000),
479 } as unknown as OrchestrationV2Run;
480 const layerTest = ThreadManagementService.layer.pipe(
481 Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
482 Layer.provide(
483 Layer.mock(Orchestrator.OrchestratorV2)({
484 getThreadEventSequence: () => Effect.succeed(0),
485 restartContinuationPending: () => Effect.succeed(false),
486 delegatedTaskResultPending: () => Effect.succeed(false),
487 streamStoredEventsFrom: () => Stream.never,
488 getThreadRecords: () =>
489 Effect.succeed({
490 thread: { id: threadId, projectId, deletedAt: null },
491 runs: [rolledBack],
492 messages: [],
493 subagents: [],
494 providerThreads: [],
495 } as unknown as OrchestrationV2ThreadProjection),
496 }),
497 ),
498 Layer.provide(ServerSettings.layerTest()),
499 Layer.provide(layerReceiptsTest),
500 );
501 const service = yield* ThreadManagementService.ThreadManagementService.pipe(
502 Effect.provide(layerTest),
503 );
504 const fiber = yield* service
505 .waitForThread({
506 projectId,
507 threadId,
508 runId,
509 timeoutMs: 60_000,
510 untilDelegatedWorkSettled: true,
511 })
512 .pipe(Effect.forkChild);
513 // Nothing will ever run there again, so a wait still open runs out here.
514 yield* TestClock.adjust(Duration.millis(60_000));
515 const result = yield* Fiber.join(fiber);
516
517 expect(result).toMatchObject({
518 timedOut: false,
519 run: { id: runId, status: "rolled_back" },
520 resultRun: { id: runId, status: "rolled_back" },
521 });
522 }),
523 );
524
525 it.effect("waitForThread returns the cut run once its continuation is declined", () =>
526 Effect.gen(function* () {
527 const projectId = ProjectId.make("project:thread-management:wait-declined");
528 const threadId = ThreadId.make("thread:thread-management:wait-declined");
529 const runId = RunId.make("run:thread-management:wait-declined");
530 const subscribed = yield* Deferred.make<void>();
531 const declined = yield* Deferred.make<void>();
532 let pending = true;
533 const layerTest = ThreadManagementService.layer.pipe(
534 Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
535 Layer.provide(
536 Layer.mock(Orchestrator.OrchestratorV2)({
537 getThreadEventSequence: () => Effect.succeed(0),
538 restartContinuationPending: () => Effect.succeed(pending),
539 awaitRestartContinuation: () => Deferred.await(declined),
540 streamStoredEventsFrom: (input) =>
541 input?.eventType === "run.updated"
542 ? Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe(
543 Stream.drain,
544 Stream.concat(Stream.never),
545 )
546 : Stream.never,
547 getThreadRecords: () =>
548 Effect.succeed({
549 thread: { id: threadId, projectId, deletedAt: null },
550 runs: [{ id: runId, status: "cancelled" }],
551 } as unknown as OrchestrationV2ThreadProjection),
552 }),
553 ),
554 Layer.provide(ServerSettings.layerTest()),
555 Layer.provide(layerReceiptsTest),
556 );
557 const service = yield* ThreadManagementService.ThreadManagementService.pipe(
558 Effect.provide(layerTest),
559 );
560 const fiber = yield* service
561 .waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 })
562 .pipe(Effect.forkChild);
563 yield* Deferred.await(subscribed);
564 pending = false;
565 yield* Deferred.succeed(declined, undefined);
566 const result = yield* Fiber.join(fiber);
567
568 expect(result).toMatchObject({ timedOut: false, run: { id: runId, status: "cancelled" } });
569 }),
570 );
571
572 it.effect.each([
573 "live",
574 "no-match",
575 "no-match-clear",
576 "no-session",
577 "unsupported-runtime",
578 ] as const)(
579 "marks background work awaited on the thread's active provider thread with state %s",
580 (state) =>
581 Effect.gen(function* () {
582 const threadId = ThreadId.make("thread:background-wait");
583 const providerThreadId = ProviderThreadId.make("provider-thread:background-wait");
584 const providerSessionId = ProviderSessionId.make("provider-session:background-wait");
585 const driver = ProviderDriverKind.make("claudeAgent");
586 const now = yield* DateTime.now;
587 const providerThread: OrchestrationV2ProviderThread = {
588 id: providerThreadId,
589 driver,
590 providerInstanceId: ProviderInstanceId.make("claudeAgent"),
591 providerSessionId,
592 appThreadId: threadId,
593 ownerNodeId: null,
594 nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" },
595 nativeConversationHeadRef: null,
596 status: "idle",
597 firstRunOrdinal: 1,
598 lastRunOrdinal: 1,
599 handoffIds: [],
600 forkedFrom: null,
601 pendingBackgroundTasks: [{ taskId: "tests", kind: "command" }],
602 contextUsage: null,
603 nativeMetadata: null,
604 createdAt: now,
605 updatedAt: now,
606 };
607 const calls: Array<{ providerThreadId: ProviderThreadId; awaited: boolean }> = [];
608 const runtime = {
609 driver,
610 providerSessionId,
611 ...(state === "unsupported-runtime"
612 ? {}
613 : {
614 awaitBackgroundTasks: (input: {
615 readonly providerThread: OrchestrationV2ProviderThread;
616 readonly taskIds: ReadonlyArray<string>;
617 readonly awaited: boolean;
618 }) =>
619 Effect.sync(() => {
620 calls.push({
621 providerThreadId: input.providerThread.id,
622 awaited: input.awaited,
623 });
624 return { matched: input.taskIds.filter((taskId) => taskId === "tests") };
625 }),
626 }),
627 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime;
628 const layerTest = ThreadManagementService.layer.pipe(
629 Layer.provide(
630 Layer.mergeAll(
631 Layer.mock(Orchestrator.OrchestratorV2)({
632 getThreadRecords: () =>
633 Effect.succeed({
634 thread: {
635 id: threadId,
636 deletedAt: null,
637 activeProviderThreadId: providerThreadId,
638 },
639 providerThreads: [providerThread],
640 } as unknown as OrchestrationV2ThreadProjection),
641 }),
642 Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
643 get: () =>
644 Effect.succeed(state === "no-session" ? Option.none() : Option.some(runtime)),
645 }),
646 ),
647 ),
648 Layer.provide(ServerSettings.layerTest()),
649 Layer.provide(layerReceiptsTest),
650 );
651 const result = yield* Effect.gen(function* () {
652 const service = yield* ThreadManagementService.ThreadManagementService;
653 return yield* service
654 .awaitBackgroundTasks({
655 threadId,
656 taskIds: state.startsWith("no-match") ? ["finished"] : ["tests", "finished"],
657 awaited: state !== "no-match-clear",
658 })
659 .pipe(Effect.result);
660 }).pipe(Effect.provide(Layer.merge(layerTest, ThreadCommandExecutor.layer)));
661
662 if (state === "live") {
663 expect(result).toMatchObject({
664 success: { matched: ["tests"], unmatched: ["finished"] },
665 });
666 expect(calls).toEqual([{ providerThreadId, awaited: true }]);
667 } else if (state === "no-match") {
668 expect(result).toMatchObject({
669 failure: { _tag: "ThreadManagementBackgroundWaitError", reason: "unmatched" },
670 });
671 expect(calls).toEqual([{ providerThreadId, awaited: true }]);
672 } else if (state === "no-match-clear") {
673 expect(result).toMatchObject({ success: { matched: [], unmatched: ["finished"] } });
674 expect(calls).toEqual([{ providerThreadId, awaited: false }]);
675 } else {
676 expect(result).toMatchObject({
677 failure: {
678 _tag: "ThreadManagementBackgroundWaitError",
679 reason: state === "no-session" ? "unavailable" : "unsupported",
680 },
681 });
682 expect(calls).toEqual([]);
683 }
684 }),
685 );
686
687 it.effect.each([
688 { status: "completed" as const, followUp: "none", settles: true },
689 { status: "failed" as const, followUp: "none", settles: false },
690 { status: "interrupted" as const, followUp: "none", settles: false },
691 { status: "completed" as const, followUp: "notification", settles: true },
692 { status: "completed" as const, followUp: "replayed-notification", settles: true },
693 { status: "completed" as const, followUp: "delegated", settles: true },
694 { status: "completed" as const, followUp: "failed-notification", settles: false },
695 { status: "completed" as const, followUp: "held-notification", settles: true },
696 { status: "completed" as const, followUp: "user", settles: false },
697 { status: "completed" as const, followUp: "steered-notification", settles: false },
698 // Send now cancels the queued run; its message joins the run that asked to settle.
699 { status: "completed" as const, followUp: "promoted-to-steer", settles: true },
700 // A delegated-completion wake cancelled unstarted because the turn already took the result.
701 { status: "completed" as const, followUp: "cancelled-delegated", settles: true },
702 { status: "completed" as const, followUp: "stopped-delegated", settles: false },
703 ])("settleAfterRun handles $status with $followUp follow-up", ({ status, followUp, settles }) =>
704 Effect.gen(function* () {
705 const projectId = ProjectId.make("project:thread-management:settle-after-run");
706 const threadId = ThreadId.make("thread:thread-management:settle-after-run");
707 const runId = RunId.make("run:thread-management:settle-after-run");
708 const followUpId = RunId.make(`${runId}:follow-up`);
709 const subscribed = yield* Deferred.make<void>();
710 const events = yield* Queue.unbounded<OrchestrationV2StoredEvent>();
711 const replayRejected = yield* Deferred.make<void>();
712 let sequence = 0;
713 let sequenceReads = 0;
714 let rejectedCommandId: CommandId | undefined;
715 const runs = [{ id: runId, ordinal: 1, status, userMessageId: MessageId.make("original") }];
716 const nextRun = {
717 id: followUpId,
718 ordinal: 2,
719 status: "running" as OrchestrationV2Run["status"],
720 userMessageId: MessageId.make("follow-up"),
721 startedAt: followUp === "stopped-delegated" ? "2026-01-01T00:00:00.000Z" : null,
722 };
723 const dispatched: Array<string> = [];
724 // The notification starts after the completed run was observed, before settlement commits.
725 let raced = false;
726 const layerTest = ThreadManagementService.layer.pipe(
727 Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
728 Layer.provide(
729 Layer.mock(Orchestrator.OrchestratorV2)({
730 getThreadEventSequence: () =>
731 Effect.sync(() => {
732 if (followUp === "replayed-notification" && sequenceReads++ === 2) sequence = 1;
733 return sequence;
734 }),
735 restartContinuationPending: () => Effect.succeed(false),
736 streamStoredEventsFrom: () =>
737 Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe(
738 Stream.drain,
739 Stream.concat(Stream.fromQueue(events)),
740 ),
741 getThreadRecords: (_threadId, _fields, filter) =>
742 Effect.sync(
743 () =>
744 ({
745 thread: { id: threadId, projectId, deletedAt: null },
746 runs: raced ? [...runs, nextRun] : runs,
747 messages: [
748 { id: MessageId.make("original"), runId, role: "user" as const },
749 {
750 id: nextRun.userMessageId,
751 runId: followUp === "promoted-to-steer" ? runId : followUpId,
752 role: "user" as const,
753 ...(followUp.endsWith("notification") ? { notification: {} } : {}),
754 ...(followUp.endsWith("delegated") ? { delegatedCompletion: {} } : {}),
755 },
756 { id: MessageId.make("reply"), runId: followUpId, role: "assistant" as const },
757 ...(followUp === "steered-notification"
758 ? [
759 {
760 id: MessageId.make("user-steer"),
761 runId: followUpId,
762 role: "user" as const,
763 },
764 ]
765 : []),
766 ].filter(
767 (message) =>
768 (filter?.messageIds === undefined ||
769 filter.messageIds.includes(message.id)) &&
770 (filter?.messageRunIds === undefined ||
771 filter.messageRunIds.includes(message.runId)) &&
772 (filter?.messageRoles === undefined ||
773 filter.messageRoles.includes(message.role)),
774 ),
775 }) as unknown as OrchestrationV2ThreadProjection,
776 ),
777 dispatch: (command) =>
778 Effect.gen(function* () {
779 if (followUp !== "none" && !raced) {
780 raced = true;
781 rejectedCommandId = command.commandId;
782 return yield* new Orchestrator.OrchestratorDispatchError({
783 commandId: command.commandId,
784 commandType: command.type,
785 cause: "Thread is still running",
786 });
787 }
788 if (followUp === "replayed-notification" && command.commandId === rejectedCommandId) {
789 yield* Deferred.succeed(replayRejected, undefined);
790 return yield* new Orchestrator.OrchestratorCommandPreviouslyRejectedError({
791 commandId: command.commandId,
792 commandType: command.type,
793 detail: "Thread is still running",
794 });
795 }
796 dispatched.push(command.type);
797 return { sequence: 1, storedEvents: [] } as never;
798 }),
799 }),
800 ),
801 Layer.provide(ServerSettings.layerTest()),
802 Layer.provide(layerReceiptsTest),
803 );
804 const service = yield* ThreadManagementService.ThreadManagementService.pipe(
805 Effect.provide(layerTest),
806 );
807 const fiber = yield* service
808 .settleAfterRun({ projectId, threadId, runId })
809 .pipe(Effect.forkChild);
810 if (followUp !== "none") {
811 yield* Deferred.await(subscribed);
812 expect(dispatched).toEqual([]);
813 nextRun.status =
814 followUp === "failed-notification"
815 ? "failed"
816 : followUp === "held-notification"
817 ? "queued"
818 : followUp === "promoted-to-steer" || followUp.endsWith("ed-delegated")
819 ? "cancelled"
820 : "completed";
821 yield* Queue.offer(events, {
822 sequence: 1,
823 event: { type: "run.updated", threadId, payload: nextRun },
824 } as unknown as OrchestrationV2StoredEvent);
825 }
826 if (followUp === "replayed-notification") {
827 yield* Deferred.await(replayRejected);
828 expect(dispatched).toEqual([]);
829 sequence = 2;
830 yield* Queue.offer(events, {
831 sequence,
832 event: { type: "run.updated", threadId, payload: nextRun },
833 } as unknown as OrchestrationV2StoredEvent);
834 }
835 yield* Fiber.join(fiber);
836 expect(dispatched).toEqual(settles ? ["thread.settle"] : []);
837 }),
838 );
839
840 it.effect.each([
841 {
842 name: "kills the thread's own agent and ends its turn",
843 others: [] as ReadonlyArray<string>,
844 killed: true,
845 expected: { outcome: "killed", sharedWithThreadIds: [] },
846 teardown: ["release"],
847 },
848 {
849 name: "keeps a process another working thread shares and only ends this turn",
850 others: ["thread:thread-management:force-stop:other"],
851 killed: false,
852 expected: {
853 outcome: "shared",
854 sharedWithThreadIds: ["thread:thread-management:force-stop:other"],
855 },
856 teardown: ["detach"],
857 },
858 {
859 name: "still settles a finished run's background work in a shared process",
860 others: ["thread:thread-management:force-stop:other"],
861 killed: false,
862 settledRun: true,
863 expected: {
864 outcome: "shared",
865 sharedWithThreadIds: ["thread:thread-management:force-stop:other"],
866 },
867 teardown: ["detach"],
868 },
869 {
870 name: "detaches when a sibling starts a turn just before the session is released",
871 others: [] as ReadonlyArray<string>,
872 killed: false,
873 siblingBusy: true,
874 expected: { outcome: "shared", sharedWithThreadIds: [] },
875 teardown: ["release", "detach"],
876 },
877 {
878 name: "reports a teardown that has not finished in time as still releasing",
879 others: [] as ReadonlyArray<string>,
880 killed: false,
881 releaseHangs: true,
882 expected: { outcome: "releasing", sharedWithThreadIds: [] },
883 teardown: ["release"],
884 },
885 ])("Force stop $name", ({ others, killed, expected, teardown, ...row }) =>
886 Effect.gen(function* () {
887 const threadId = ThreadId.make("thread:thread-management:force-stop");
888 const providerSessionId = ProviderSessionId.make("provider-session:force-stop");
889 const providerThreadId = ProviderThreadId.make("provider-thread:force-stop");
890 const runId = RunId.make("run:thread-management:force-stop");
891 const dispatched: Array<OrchestrationV2Command | { readonly type: string }> = [];
892 const teardowns: Array<string> = [];
893 const tornDown = yield* Deferred.make<void>();
894 const layerTest = ThreadManagementService.layer.pipe(
895 Layer.provide(
896 Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
897 isLive: () => Effect.succeed(true),
898 otherWorkingThreads: () => Effect.succeed(others.map((id) => ThreadId.make(id))),
899 releaseUnlessBusy: () =>
900 Effect.sync(() => teardowns.push("release")).pipe(
901 Effect.andThen(
902 "releaseHangs" in row ? Effect.never : Effect.succeed(!("siblingBusy" in row)),
903 ),
904 ),
905 detach: () =>
906 Effect.sync(() => teardowns.push("detach")).pipe(
907 Effect.andThen(Deferred.succeed(tornDown, undefined)),
908 Effect.asVoid,
909 ),
910 }),
911 ),
912 Layer.provide(
913 Layer.mock(Orchestrator.OrchestratorV2)({
914 getThreadRecords: () =>
915 Effect.succeed({
916 thread: { id: threadId, deletedAt: null },
917 runs: [
918 {
919 id: runId,
920 ordinal: 1,
921 status: "settledRun" in row ? "completed" : "running",
922 activeAttemptId: "attempt",
923 },
924 // A queued follow-up neither supersedes the request nor becomes the target.
925 { id: RunId.make("run:queued-follow-up"), ordinal: 2, status: "queued" },
926 ],
927 providerThreads: [{ id: providerThreadId, appThreadId: threadId, providerSessionId }],
928 providerTurns: [
929 {
930 id: "provider-turn:force-stop",
931 providerThreadId,
932 runAttemptId: "attempt",
933 status: "settledRun" in row ? "completed" : "running",
934 },
935 ],
936 } as unknown as OrchestrationV2ThreadProjection),
937 dispatch: (command) =>
938 Effect.sync(() => {
939 dispatched.push(command);
940 return { sequence: 1, storedEvents: [] } as never;
941 }),
942 }),
943 ),
944 Layer.provide(
945 Layer.succeed(AgentScope, {
946 ...AgentScope.defaultValue(),
947 kill: () => Effect.succeed(killed),
948 }),
949 ),
950 Layer.provide(ServerSettings.layerTest()),
951 Layer.provide(layerReceiptsTest),
952 );
953 yield* Effect.gen(function* () {
954 const service = yield* ThreadManagementService.ThreadManagementService;
955 const stopping = yield* service.forceStopThread({ threadId, runId }).pipe(Effect.forkChild);
956 yield* TestClock.adjust("10 seconds");
957 expect(yield* Fiber.join(stopping)).toEqual(expected);
958 expect(dispatched).toMatchObject([
959 {
960 type: "thread.background-work.settle",
961 providerThreadId,
962 reason: "Force stopped by user",
963 },
964 ]);
965 if (teardown.at(-1) === "detach") yield* Deferred.await(tornDown);
966 expect(teardowns).toEqual(teardown);
967 }).pipe(Effect.provide(layerTest));
968 }),
969 );
970
971 it.effect("Force stop refuses once a newer run has started, before killing anything", () =>
972 Effect.gen(function* () {
973 const threadId = ThreadId.make("thread:thread-management:force-stop-superseded");
974 let killed = false;
975 const layerTest = ThreadManagementService.layer.pipe(
976 Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
977 Layer.provide(
978 Layer.mock(Orchestrator.OrchestratorV2)({
979 getThreadRecords: () =>
980 Effect.succeed({
981 thread: { id: threadId, deletedAt: null },
982 runs: [
983 { id: RunId.make("run:stopped"), ordinal: 1, status: "running" },
984 { id: RunId.make("run:newer"), ordinal: 2, status: "running" },
985 ],
986 providerThreads: [],
987 providerTurns: [],
988 } as unknown as OrchestrationV2ThreadProjection),
989 }),
990 ),
991 Layer.provide(
992 Layer.succeed(AgentScope, {
993 ...AgentScope.defaultValue(),
994 kill: () => Effect.sync(() => (killed = true)),
995 }),
996 ),
997 Layer.provide(ServerSettings.layerTest()),
998 Layer.provide(layerReceiptsTest),
999 );
1000 const error = yield* Effect.gen(function* () {
1001 const service = yield* ThreadManagementService.ThreadManagementService;
1002 return yield* Effect.flip(
1003 service.forceStopThread({ threadId, runId: RunId.make("run:stopped") }),
1004 );
1005 }).pipe(Effect.provide(layerTest));
1006 expect(error.reason).toBe("superseded");
1007 expect(killed).toBe(false);
1008 }),
1009 );
1010
1011 it.effect(
1012 "admits two transcript exports at a time and frees a slot on failure or interruption",
1013 () =>
1014 Effect.gen(function* () {
1015 const started = yield* Queue.unbounded<string>();
1016 const gates = new Map(
1017 ["a", "b", "c", "d", "e", "f"].map((id) => [
1018 id,
1019 Deferred.makeUnsafe<void, Orchestrator.OrchestratorV2Error>(),
1020 ]),
1021 );
1022 const layerTest = ThreadManagementService.layer.pipe(
1023 Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
1024 Layer.provide(
1025 Layer.mock(Orchestrator.OrchestratorV2)({
1026 getTranscriptByteFloor: () => Effect.succeed(0),
1027 getThreadSnapshot: (threadId) =>
1028 Queue.offer(started, threadId).pipe(
1029 Effect.andThen(Deferred.await(gates.get(threadId)!)),
1030 Effect.as({
1031 schemaVersion: 1,
1032 snapshotSequence: 1,
1033 projection: {
1034 thread: { id: threadId, title: threadId },
1035 updatedAt: DateTime.makeUnsafe(0),
1036 visibleTurnItems: [],
1037 } as unknown as OrchestrationV2ThreadProjection,
1038 }),
1039 ),
1040 }),
1041 ),
1042 Layer.provide(ServerSettings.layerTest()),
1043 Layer.provide(layerReceiptsTest),
1044 );
1045
1046 yield* Effect.gen(function* () {
1047 const threads = yield* ThreadManagementService.ThreadManagementService;
1048 const exportThread = (id: string) =>
1049 threads.getThreadTranscript(ThreadId.make(id)).pipe(
1050 Effect.provideService(ServerEnvironment.ServerEnvironmentIdentity, {
1051 getEnvironmentId: Effect.succeed(EnvironmentId.make("environment:export")),
1052 }),
1053 Effect.forkChild({ startImmediately: true }),
1054 );
1055 const a = yield* exportThread("a");
1056 const b = yield* exportThread("b");
1057 const c = yield* exportThread("c");
1058 const d = yield* exportThread("d");
1059 const e = yield* exportThread("e");
1060 const f = yield* exportThread("f");
1061 expect(yield* Queue.takeAll(started)).toEqual(["a", "b"]);
1062
1063 yield* Fiber.interrupt(c);
1064 yield* Deferred.fail(
1065 gates.get("a")!,
1066 new Orchestrator.OrchestratorProjectionError({ threadId: ThreadId.make("a"), cause: "" }),
1067 );
1068 expect(yield* Queue.take(started)).toBe("d");
1069 yield* Fiber.interrupt(b);
1070 expect(yield* Queue.take(started)).toBe("e");
1071 yield* Deferred.succeed(gates.get("d")!, undefined);
1072 expect(yield* Queue.take(started)).toBe("f");
1073 yield* Deferred.succeed(gates.get("e")!, undefined);
1074 yield* Deferred.succeed(gates.get("f")!, undefined);
1075
1076 expect((yield* Fiber.await(a))._tag).toBe("Failure");
1077 expect((yield* Fiber.join(d)).threadId).toBe("d");
1078 expect((yield* Fiber.join(e)).threadId).toBe("e");
1079 expect((yield* Fiber.join(f)).threadId).toBe("f");
1080 expect(yield* Queue.size(started)).toBe(0);
1081 }).pipe(Effect.provide(layerTest));
1082 }),
1083 );