The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { expect, it } from "@effect/vitest"; |
| 2 | import { |
| 3 | CommandId, |
| 4 | EnvironmentId, |
| 5 | MessageId, |
| 6 | type OrchestrationV2Command, |
| 7 | type OrchestrationV2Run, |
| 8 | type OrchestrationV2ProviderThread, |
| 9 | ProviderThreadId, |
| 10 | ProviderDriverKind, |
| 11 | ProviderSessionId, |
| 12 | type OrchestrationV2StoredEvent, |
| 13 | type OrchestrationV2ThreadProjection, |
| 14 | ProjectId, |
| 15 | ProviderInstanceId, |
| 16 | RunId, |
| 17 | ThreadId, |
| 18 | } from "@t3tools/contracts"; |
| 19 | import * as Deferred from "effect/Deferred"; |
| 20 | import * as DateTime from "effect/DateTime"; |
| 21 | import * as Duration from "effect/Duration"; |
| 22 | import * as Option from "effect/Option"; |
| 23 | import * as Effect from "effect/Effect"; |
| 24 | import * as Fiber from "effect/Fiber"; |
| 25 | import * as Layer from "effect/Layer"; |
| 26 | import * as Queue from "effect/Queue"; |
| 27 | import * as Stream from "effect/Stream"; |
| 28 | import * as TestClock from "effect/testing/TestClock"; |
| 29 | |
| 30 | import { AgentScope } from "@t3tools/shared/AgentScope"; |
| 31 | |
| 32 | import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; |
| 33 | import * as ServerSettings from "../serverSettings.ts"; |
| 34 | import * as SqlitePersistence from "../persistence/Sqlite.ts"; |
| 35 | import * as CommandReceiptStore from "./CommandReceiptStore.ts"; |
| 36 | import * as Orchestrator from "./Orchestrator.ts"; |
| 37 | import * as ProviderSessionManager from "./ProviderSessionManager.ts"; |
| 38 | import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; |
| 39 | import * as ThreadManagementService from "./ThreadManagementService.ts"; |
| 40 | import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts"; |
| 41 | import * as LegacyV1ThreadImporter from "./legacy/LegacyV1ThreadImporter.ts"; |
| 42 | import { |
| 43 | ThreadWorktreeRestore, |
| 44 | WorktreeRecreateError, |
| 45 | } from "../workspace/restoreMissingWorktree.ts"; |
| 46 | |
| 47 | it("stamps authoritative provenance on commands that create threads or messages", () => { |
| 48 | const command: OrchestrationV2Command = { |
| 49 | type: "thread.create", |
| 50 | createdBy: "agent", |
| 51 | creationSource: "mcp", |
| 52 | commandId: CommandId.make("command:thread-management:create"), |
| 53 | threadId: ThreadId.make("thread:thread-management:create"), |
| 54 | projectId: ProjectId.make("project:thread-management"), |
| 55 | title: "Thread management", |
| 56 | modelSelection: { |
| 57 | instanceId: ProviderInstanceId.make("codex"), |
| 58 | model: "gpt-5-codex", |
| 59 | }, |
| 60 | runtimeMode: "full-access", |
| 61 | interactionMode: "default", |
| 62 | branch: null, |
| 63 | worktreePath: null, |
| 64 | // A client cannot claim a linked environment's origin, or a parent there. |
| 65 | linkOrigin: { sessionId: "forged", label: "Somewhere else" }, |
| 66 | delegatedFrom: { |
| 67 | environmentId: EnvironmentId.make("environment:elsewhere"), |
| 68 | threadId: ThreadId.make("thread:elsewhere"), |
| 69 | title: "Somewhere else", |
| 70 | }, |
| 71 | // A client's message goes through intake; only the server folds one into the creation. |
| 72 | initialMessage: { |
| 73 | messageId: MessageId.make("message:thread-management:forged"), |
| 74 | text: "Forged", |
| 75 | attachments: [], |
| 76 | }, |
| 77 | }; |
| 78 | |
| 79 | const stamped = ThreadManagementService.withCreationProvenance(command, { |
| 80 | createdBy: "user", |
| 81 | creationSource: "web", |
| 82 | }); |
| 83 | expect(stamped).toMatchObject({ |
| 84 | createdBy: "user", |
| 85 | creationSource: "web", |
| 86 | }); |
| 87 | expect(stamped).not.toHaveProperty("linkOrigin"); |
| 88 | expect(stamped).not.toHaveProperty("delegatedFrom"); |
| 89 | expect(stamped).not.toHaveProperty("initialMessage"); |
| 90 | }); |
| 91 | |
| 92 | const layerReceiptsTest = CommandReceiptStore.layer.pipe( |
| 93 | Layer.provide(SqlitePersistence.layerMemory), |
| 94 | ); |
| 95 | |
| 96 | const layerDispatchChokepointTest = |
| 97 | (settingsLayer: ReturnType<typeof ServerSettings.layerTest>) => (dispatched: Array<string>) => |
| 98 | ThreadManagementService.layer.pipe( |
| 99 | Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), |
| 100 | Layer.provide( |
| 101 | Layer.mock(Orchestrator.OrchestratorV2)({ |
| 102 | getThreadRecords: () => |
| 103 | Effect.succeed({ |
| 104 | thread: { |
| 105 | id: ThreadId.make("thread:thread-management:chokepoint"), |
| 106 | projectId: ProjectId.make("project:thread-management:chokepoint"), |
| 107 | deletedAt: null, |
| 108 | modelSelection: { |
| 109 | instanceId: ProviderInstanceId.make("codex_work"), |
| 110 | model: "test", |
| 111 | }, |
| 112 | }, |
| 113 | } as unknown as OrchestrationV2ThreadProjection), |
| 114 | dispatch: (command) => |
| 115 | Effect.sync(() => { |
| 116 | dispatched.push(command.type); |
| 117 | return { sequence: 1, storedEvents: [] } as never; |
| 118 | }), |
| 119 | }), |
| 120 | ), |
| 121 | Layer.provide(settingsLayer), |
| 122 | Layer.provideMerge(layerReceiptsTest), |
| 123 | ); |
| 124 | |
| 125 | it.effect( |
| 126 | "the dispatch chokepoint refuses a continuation message.dispatch on a project-disabled instance before reaching the orchestrator", |
| 127 | () => { |
| 128 | const projectId = ProjectId.make("project:thread-management:chokepoint"); |
| 129 | const threadId = ThreadId.make("thread:thread-management:chokepoint"); |
| 130 | const instanceId = ProviderInstanceId.make("codex_work"); |
| 131 | const commandId = CommandId.make("command:thread-management:chokepoint-disabled"); |
| 132 | const dispatched: Array<string> = []; |
| 133 | const layerTest = layerDispatchChokepointTest( |
| 134 | ServerSettings.layerTest({ |
| 135 | providerInstances: { [instanceId]: { driver: "codex", enabled: true } }, |
| 136 | projectSettingsOverrides: { |
| 137 | [projectId]: { providerInstances: { [instanceId]: { enabled: false } } }, |
| 138 | }, |
| 139 | }), |
| 140 | )(dispatched); |
| 141 | |
| 142 | return Effect.gen(function* () { |
| 143 | const threads = yield* ThreadManagementService.ThreadManagementService; |
| 144 | const error = yield* Effect.flip( |
| 145 | threads.dispatch({ |
| 146 | type: "message.dispatch", |
| 147 | commandId, |
| 148 | threadId, |
| 149 | messageId: MessageId.make("message:thread-management:chokepoint-disabled"), |
| 150 | text: "Background task completed.", |
| 151 | attachments: [], |
| 152 | dispatchMode: { type: "queue_after_active" }, |
| 153 | createdBy: "agent", |
| 154 | creationSource: "server", |
| 155 | }), |
| 156 | ); |
| 157 | |
| 158 | expect(error).toBeInstanceOf( |
| 159 | ThreadManagementService.ThreadManagementProviderInstanceDisabledError, |
| 160 | ); |
| 161 | expect(error).toMatchObject({ projectId, instanceId }); |
| 162 | expect(dispatched).toEqual([]); |
| 163 | }).pipe(Effect.provide(layerTest)); |
| 164 | }, |
| 165 | ); |
| 166 | |
| 167 | it.effect( |
| 168 | "a replay of an already-accepted command id dispatches despite the instance now being disabled", |
| 169 | () => { |
| 170 | const projectId = ProjectId.make("project:thread-management:chokepoint"); |
| 171 | const threadId = ThreadId.make("thread:thread-management:chokepoint"); |
| 172 | const instanceId = ProviderInstanceId.make("codex_work"); |
| 173 | const commandId = CommandId.make("command:thread-management:chokepoint-replay"); |
| 174 | const dispatched: Array<string> = []; |
| 175 | const layerTest = layerDispatchChokepointTest( |
| 176 | ServerSettings.layerTest({ |
| 177 | providerInstances: { [instanceId]: { driver: "codex", enabled: true } }, |
| 178 | projectSettingsOverrides: { |
| 179 | [projectId]: { providerInstances: { [instanceId]: { enabled: false } } }, |
| 180 | }, |
| 181 | }), |
| 182 | )(dispatched); |
| 183 | |
| 184 | return Effect.gen(function* () { |
| 185 | const receipts = yield* CommandReceiptStore.CommandReceiptStoreV2; |
| 186 | yield* receipts.insertIfAbsent({ |
| 187 | commandId, |
| 188 | threadId, |
| 189 | commandType: "message.dispatch", |
| 190 | acceptedAt: yield* DateTime.now, |
| 191 | resultSequence: 1, |
| 192 | status: "accepted", |
| 193 | error: null, |
| 194 | }); |
| 195 | |
| 196 | const threads = yield* ThreadManagementService.ThreadManagementService; |
| 197 | yield* threads.dispatch({ |
| 198 | type: "message.dispatch", |
| 199 | commandId, |
| 200 | threadId, |
| 201 | messageId: MessageId.make("message:thread-management:chokepoint-replay"), |
| 202 | text: "Background task completed.", |
| 203 | attachments: [], |
| 204 | dispatchMode: { type: "queue_after_active" }, |
| 205 | createdBy: "agent", |
| 206 | creationSource: "server", |
| 207 | }); |
| 208 | |
| 209 | expect(dispatched).toEqual(["message.dispatch"]); |
| 210 | }).pipe(Effect.provide(layerTest)); |
| 211 | }, |
| 212 | ); |
| 213 | |
| 214 | it.effect.each([ |
| 215 | { label: "fails a send that starts a turn", running: false }, |
| 216 | { label: "still steers a running turn", running: true }, |
| 217 | ])("sendToThread with a missing worktree that cannot be recreated $label", ({ running }) => { |
| 218 | const projectId = ProjectId.make("project:thread-management:worktree-gone"); |
| 219 | const threadId = ThreadId.make("thread:thread-management:worktree-gone"); |
| 220 | const dispatched: Array<string> = []; |
| 221 | const layerTest = ThreadManagementService.layerWithLegacyImporter.pipe( |
| 222 | Layer.provide( |
| 223 | Layer.succeed(ThreadWorktreeRestore, { |
| 224 | restore: () => |
| 225 | Effect.fail( |
| 226 | new WorktreeRecreateError({ |
| 227 | worktreePath: "/worktrees/gone", |
| 228 | branch: "t3/gone", |
| 229 | cause: new Error("branch_checked_out_in_worktree"), |
| 230 | }), |
| 231 | ), |
| 232 | }), |
| 233 | ), |
| 234 | Layer.provide( |
| 235 | Layer.mock(LegacyV1ThreadImporter.LegacyV1ThreadImporter)({ |
| 236 | ensureTranscript: () => Effect.succeed({ importedThreadCount: 0, importedMessageCount: 0 }), |
| 237 | }), |
| 238 | ), |
| 239 | Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), |
| 240 | Layer.provide( |
| 241 | Layer.mock(Orchestrator.OrchestratorV2)({ |
| 242 | getThreadRecords: () => |
| 243 | Effect.succeed({ |
| 244 | thread: { |
| 245 | id: threadId, |
| 246 | projectId, |
| 247 | deletedAt: null, |
| 248 | archivedAt: null, |
| 249 | worktreePath: "/worktrees/gone", |
| 250 | branch: "t3/gone", |
| 251 | modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "test" }, |
| 252 | }, |
| 253 | runs: running |
| 254 | ? [ |
| 255 | { |
| 256 | id: RunId.make("run:thread-management:worktree-gone"), |
| 257 | status: "running", |
| 258 | activeAttemptId: "attempt:thread-management:worktree-gone", |
| 259 | }, |
| 260 | ] |
| 261 | : [], |
| 262 | providerTurns: running |
| 263 | ? [ |
| 264 | { |
| 265 | runAttemptId: "attempt:thread-management:worktree-gone", |
| 266 | status: "running", |
| 267 | }, |
| 268 | ] |
| 269 | : [], |
| 270 | messages: [], |
| 271 | turnItems: [], |
| 272 | } as unknown as OrchestrationV2ThreadProjection), |
| 273 | dispatch: (command) => |
| 274 | Effect.sync(() => { |
| 275 | dispatched.push(command.type); |
| 276 | return { sequence: 1, storedEvents: [] } as never; |
| 277 | }), |
| 278 | }), |
| 279 | ), |
| 280 | Layer.provide(ServerSettings.layerTest()), |
| 281 | Layer.provideMerge(layerReceiptsTest), |
| 282 | ); |
| 283 | |
| 284 | return Effect.gen(function* () { |
| 285 | const threads = yield* ThreadManagementService.ThreadManagementService; |
| 286 | const error = yield* Effect.flip( |
| 287 | threads.sendToThread({ |
| 288 | projectId, |
| 289 | threadId, |
| 290 | commandId: CommandId.make("command:thread-management:worktree-gone"), |
| 291 | messageId: MessageId.make("message:thread-management:worktree-gone"), |
| 292 | text: "Reply with ok.", |
| 293 | attachments: [], |
| 294 | mode: "auto", |
| 295 | createdBy: "agent", |
| 296 | creationSource: "mcp", |
| 297 | }), |
| 298 | ); |
| 299 | // The mock records no message, so a dispatched send ends in the projection read. |
| 300 | expect(error._tag).toBe( |
| 301 | running |
| 302 | ? "ThreadManagementDurableRunProjectionError" |
| 303 | : "ThreadManagementWorktreeRestoreError", |
| 304 | ); |
| 305 | expect(dispatched).toEqual(running ? ["message.dispatch"] : []); |
| 306 | }).pipe(Effect.provide(layerTest)); |
| 307 | }); |
| 308 | |
| 309 | it.effect("waitForThread follows a restart continuation instead of the cut run", () => |
| 310 | Effect.gen(function* () { |
| 311 | const projectId = ProjectId.make("project:thread-management:wait-restart"); |
| 312 | const threadId = ThreadId.make("thread:thread-management:wait-restart"); |
| 313 | const runId = RunId.make("run:thread-management:wait-restart"); |
| 314 | const continuationId = RunId.make("run:thread-management:wait-restart:2"); |
| 315 | const subscribed = yield* Deferred.make<void>(); |
| 316 | const events = yield* Queue.unbounded<OrchestrationV2StoredEvent>(); |
| 317 | let runs: Array<{ |
| 318 | id: RunId; |
| 319 | status: OrchestrationV2Run["status"]; |
| 320 | restartContinuationOfRunId?: RunId; |
| 321 | }> = [{ id: runId, status: "cancelled" }]; |
| 322 | let pending = true; |
| 323 | const stored = (sequence: number, event: object) => |
| 324 | ({ sequence, event: { threadId, ...event } }) as unknown as OrchestrationV2StoredEvent; |
| 325 | const layerTest = ThreadManagementService.layer.pipe( |
| 326 | Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), |
| 327 | Layer.provide( |
| 328 | Layer.mock(Orchestrator.OrchestratorV2)({ |
| 329 | getThreadEventSequence: () => Effect.succeed(0), |
| 330 | restartContinuationPending: (run) => Effect.succeed(pending && run.id === runId), |
| 331 | awaitRestartContinuation: () => Effect.never, |
| 332 | streamStoredEventsFrom: (input) => |
| 333 | input?.eventType === "run.updated" |
| 334 | ? Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe( |
| 335 | Stream.drain, |
| 336 | Stream.concat(Stream.fromQueue(events)), |
| 337 | ) |
| 338 | : Stream.never, |
| 339 | getThreadRecords: () => |
| 340 | Effect.sync( |
| 341 | () => |
| 342 | ({ |
| 343 | thread: { id: threadId, projectId, deletedAt: null }, |
| 344 | runs, |
| 345 | }) as unknown as OrchestrationV2ThreadProjection, |
| 346 | ), |
| 347 | }), |
| 348 | ), |
| 349 | Layer.provide(ServerSettings.layerTest()), |
| 350 | Layer.provide(layerReceiptsTest), |
| 351 | ); |
| 352 | const service = yield* ThreadManagementService.ThreadManagementService.pipe( |
| 353 | Effect.provide(layerTest), |
| 354 | ); |
| 355 | const fiber = yield* service |
| 356 | .waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 }) |
| 357 | .pipe(Effect.forkChild); |
| 358 | yield* Deferred.await(subscribed); |
| 359 | const continuation = { id: continuationId, restartContinuationOfRunId: runId }; |
| 360 | runs = [...runs, { ...continuation, status: "running" }]; |
| 361 | pending = false; |
| 362 | yield* Queue.offer( |
| 363 | events, |
| 364 | stored(1, { type: "run.updated", payload: { ...continuation, status: "running" } }), |
| 365 | ); |
| 366 | runs = [runs[0]!, { ...continuation, status: "completed" }]; |
| 367 | yield* Queue.offer( |
| 368 | events, |
| 369 | stored(2, { type: "run.updated", payload: { ...continuation, status: "completed" } }), |
| 370 | ); |
| 371 | const result = yield* Fiber.join(fiber); |
| 372 | |
| 373 | expect(result).toMatchObject({ |
| 374 | timedOut: false, |
| 375 | run: { id: continuationId, status: "completed" }, |
| 376 | }); |
| 377 | }), |
| 378 | ); |
| 379 | |
| 380 | it.effect("waitForThread can wait past an ended run for the thread's delegated work", () => |
| 381 | Effect.gen(function* () { |
| 382 | const projectId = ProjectId.make("project:thread-management:wait-delegated"); |
| 383 | const threadId = ThreadId.make("thread:thread-management:wait-delegated"); |
| 384 | const runId = RunId.make("run:thread-management:wait-delegated"); |
| 385 | const wakeRunId = RunId.make("run:thread-management:wait-delegated:wake"); |
| 386 | const subscribed = yield* Deferred.make<void>(); |
| 387 | const events = yield* Queue.unbounded<OrchestrationV2StoredEvent>(); |
| 388 | const at = (seconds: number) => DateTime.makeUnsafe(seconds * 1_000); |
| 389 | const run = (id: RunId, ordinal: number, status: OrchestrationV2Run["status"], end: number) => |
| 390 | ({ |
| 391 | id, |
| 392 | ordinal, |
| 393 | status, |
| 394 | startedAt: at(end - 1), |
| 395 | completedAt: status === "running" ? null : at(end), |
| 396 | }) as unknown as OrchestrationV2Run; |
| 397 | // The run ended, but a task it delegated still runs and will wake the thread. |
| 398 | let runs = [run(runId, 1, "completed", 1)]; |
| 399 | let subagents = [{ status: "running" }]; |
| 400 | const stored = (sequence: number, event: object) => |
| 401 | ({ sequence, event: { threadId, ...event } }) as unknown as OrchestrationV2StoredEvent; |
| 402 | const layerTest = ThreadManagementService.layer.pipe( |
| 403 | Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), |
| 404 | Layer.provide( |
| 405 | Layer.mock(Orchestrator.OrchestratorV2)({ |
| 406 | getThreadEventSequence: () => Effect.succeed(0), |
| 407 | restartContinuationPending: () => Effect.succeed(false), |
| 408 | delegatedTaskResultPending: () => Effect.succeed(false), |
| 409 | streamStoredEventsFrom: (input) => |
| 410 | input?.eventType === "subagent.updated" |
| 411 | ? Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe( |
| 412 | Stream.drain, |
| 413 | Stream.concat(Stream.fromQueue(events)), |
| 414 | ) |
| 415 | : Stream.never, |
| 416 | getThreadRecords: () => |
| 417 | Effect.sync( |
| 418 | () => |
| 419 | ({ |
| 420 | thread: { id: threadId, projectId, deletedAt: null }, |
| 421 | runs, |
| 422 | messages: [], |
| 423 | subagents, |
| 424 | providerThreads: [], |
| 425 | }) as unknown as OrchestrationV2ThreadProjection, |
| 426 | ), |
| 427 | }), |
| 428 | ), |
| 429 | Layer.provide(ServerSettings.layerTest()), |
| 430 | Layer.provide(layerReceiptsTest), |
| 431 | ); |
| 432 | const service = yield* ThreadManagementService.ThreadManagementService.pipe( |
| 433 | Effect.provide(layerTest), |
| 434 | ); |
| 435 | // Without the option, the ended run is the answer, as before. |
| 436 | expect( |
| 437 | yield* service.waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 }), |
| 438 | ).toMatchObject({ timedOut: false, run: { id: runId } }); |
| 439 | expect( |
| 440 | yield* service.waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 }), |
| 441 | ).not.toHaveProperty("resultRun"); |
| 442 | |
| 443 | const fiber = yield* service |
| 444 | .waitForThread({ |
| 445 | projectId, |
| 446 | threadId, |
| 447 | runId, |
| 448 | timeoutMs: 60 * 60 * 1_000, |
| 449 | untilDelegatedWorkSettled: true, |
| 450 | }) |
| 451 | .pipe(Effect.forkChild); |
| 452 | // A wait that returns early fails below instead of hanging here. |
| 453 | yield* Effect.raceFirst(Deferred.await(subscribed), Fiber.await(fiber)); |
| 454 | // The task finishes; its delivery wakes the thread, whose next run answers. |
| 455 | subagents = [{ status: "completed" }]; |
| 456 | runs = [...runs, run(wakeRunId, 2, "completed", 3)]; |
| 457 | yield* Queue.offer(events, stored(1, { type: "subagent.updated", payload: {} })); |
| 458 | const result = yield* Fiber.join(fiber); |
| 459 | |
| 460 | expect(result).toMatchObject({ |
| 461 | timedOut: false, |
| 462 | run: { id: runId, status: "completed" }, |
| 463 | resultRun: { id: wakeRunId, status: "completed" }, |
| 464 | }); |
| 465 | }), |
| 466 | ); |
| 467 | |
| 468 | it.effect("waitForThread settles delegated work on a thread rolled back past its only run", () => |
| 469 | Effect.gen(function* () { |
| 470 | const projectId = ProjectId.make("project:thread-management:wait-rolled-back"); |
| 471 | const threadId = ThreadId.make("thread:thread-management:wait-rolled-back"); |
| 472 | const runId = RunId.make("run:thread-management:wait-rolled-back"); |
| 473 | const rolledBack = { |
| 474 | id: runId, |
| 475 | ordinal: 1, |
| 476 | status: "rolled_back", |
| 477 | startedAt: DateTime.makeUnsafe(0), |
| 478 | completedAt: DateTime.makeUnsafe(1_000), |
| 479 | } as unknown as OrchestrationV2Run; |
| 480 | const layerTest = ThreadManagementService.layer.pipe( |
| 481 | Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), |
| 482 | Layer.provide( |
| 483 | Layer.mock(Orchestrator.OrchestratorV2)({ |
| 484 | getThreadEventSequence: () => Effect.succeed(0), |
| 485 | restartContinuationPending: () => Effect.succeed(false), |
| 486 | delegatedTaskResultPending: () => Effect.succeed(false), |
| 487 | streamStoredEventsFrom: () => Stream.never, |
| 488 | getThreadRecords: () => |
| 489 | Effect.succeed({ |
| 490 | thread: { id: threadId, projectId, deletedAt: null }, |
| 491 | runs: [rolledBack], |
| 492 | messages: [], |
| 493 | subagents: [], |
| 494 | providerThreads: [], |
| 495 | } as unknown as OrchestrationV2ThreadProjection), |
| 496 | }), |
| 497 | ), |
| 498 | Layer.provide(ServerSettings.layerTest()), |
| 499 | Layer.provide(layerReceiptsTest), |
| 500 | ); |
| 501 | const service = yield* ThreadManagementService.ThreadManagementService.pipe( |
| 502 | Effect.provide(layerTest), |
| 503 | ); |
| 504 | const fiber = yield* service |
| 505 | .waitForThread({ |
| 506 | projectId, |
| 507 | threadId, |
| 508 | runId, |
| 509 | timeoutMs: 60_000, |
| 510 | untilDelegatedWorkSettled: true, |
| 511 | }) |
| 512 | .pipe(Effect.forkChild); |
| 513 | // Nothing will ever run there again, so a wait still open runs out here. |
| 514 | yield* TestClock.adjust(Duration.millis(60_000)); |
| 515 | const result = yield* Fiber.join(fiber); |
| 516 | |
| 517 | expect(result).toMatchObject({ |
| 518 | timedOut: false, |
| 519 | run: { id: runId, status: "rolled_back" }, |
| 520 | resultRun: { id: runId, status: "rolled_back" }, |
| 521 | }); |
| 522 | }), |
| 523 | ); |
| 524 | |
| 525 | it.effect("waitForThread returns the cut run once its continuation is declined", () => |
| 526 | Effect.gen(function* () { |
| 527 | const projectId = ProjectId.make("project:thread-management:wait-declined"); |
| 528 | const threadId = ThreadId.make("thread:thread-management:wait-declined"); |
| 529 | const runId = RunId.make("run:thread-management:wait-declined"); |
| 530 | const subscribed = yield* Deferred.make<void>(); |
| 531 | const declined = yield* Deferred.make<void>(); |
| 532 | let pending = true; |
| 533 | const layerTest = ThreadManagementService.layer.pipe( |
| 534 | Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), |
| 535 | Layer.provide( |
| 536 | Layer.mock(Orchestrator.OrchestratorV2)({ |
| 537 | getThreadEventSequence: () => Effect.succeed(0), |
| 538 | restartContinuationPending: () => Effect.succeed(pending), |
| 539 | awaitRestartContinuation: () => Deferred.await(declined), |
| 540 | streamStoredEventsFrom: (input) => |
| 541 | input?.eventType === "run.updated" |
| 542 | ? Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe( |
| 543 | Stream.drain, |
| 544 | Stream.concat(Stream.never), |
| 545 | ) |
| 546 | : Stream.never, |
| 547 | getThreadRecords: () => |
| 548 | Effect.succeed({ |
| 549 | thread: { id: threadId, projectId, deletedAt: null }, |
| 550 | runs: [{ id: runId, status: "cancelled" }], |
| 551 | } as unknown as OrchestrationV2ThreadProjection), |
| 552 | }), |
| 553 | ), |
| 554 | Layer.provide(ServerSettings.layerTest()), |
| 555 | Layer.provide(layerReceiptsTest), |
| 556 | ); |
| 557 | const service = yield* ThreadManagementService.ThreadManagementService.pipe( |
| 558 | Effect.provide(layerTest), |
| 559 | ); |
| 560 | const fiber = yield* service |
| 561 | .waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 }) |
| 562 | .pipe(Effect.forkChild); |
| 563 | yield* Deferred.await(subscribed); |
| 564 | pending = false; |
| 565 | yield* Deferred.succeed(declined, undefined); |
| 566 | const result = yield* Fiber.join(fiber); |
| 567 | |
| 568 | expect(result).toMatchObject({ timedOut: false, run: { id: runId, status: "cancelled" } }); |
| 569 | }), |
| 570 | ); |
| 571 | |
| 572 | it.effect.each([ |
| 573 | "live", |
| 574 | "no-match", |
| 575 | "no-match-clear", |
| 576 | "no-session", |
| 577 | "unsupported-runtime", |
| 578 | ] as const)( |
| 579 | "marks background work awaited on the thread's active provider thread with state %s", |
| 580 | (state) => |
| 581 | Effect.gen(function* () { |
| 582 | const threadId = ThreadId.make("thread:background-wait"); |
| 583 | const providerThreadId = ProviderThreadId.make("provider-thread:background-wait"); |
| 584 | const providerSessionId = ProviderSessionId.make("provider-session:background-wait"); |
| 585 | const driver = ProviderDriverKind.make("claudeAgent"); |
| 586 | const now = yield* DateTime.now; |
| 587 | const providerThread: OrchestrationV2ProviderThread = { |
| 588 | id: providerThreadId, |
| 589 | driver, |
| 590 | providerInstanceId: ProviderInstanceId.make("claudeAgent"), |
| 591 | providerSessionId, |
| 592 | appThreadId: threadId, |
| 593 | ownerNodeId: null, |
| 594 | nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" }, |
| 595 | nativeConversationHeadRef: null, |
| 596 | status: "idle", |
| 597 | firstRunOrdinal: 1, |
| 598 | lastRunOrdinal: 1, |
| 599 | handoffIds: [], |
| 600 | forkedFrom: null, |
| 601 | pendingBackgroundTasks: [{ taskId: "tests", kind: "command" }], |
| 602 | contextUsage: null, |
| 603 | nativeMetadata: null, |
| 604 | createdAt: now, |
| 605 | updatedAt: now, |
| 606 | }; |
| 607 | const calls: Array<{ providerThreadId: ProviderThreadId; awaited: boolean }> = []; |
| 608 | const runtime = { |
| 609 | driver, |
| 610 | providerSessionId, |
| 611 | ...(state === "unsupported-runtime" |
| 612 | ? {} |
| 613 | : { |
| 614 | awaitBackgroundTasks: (input: { |
| 615 | readonly providerThread: OrchestrationV2ProviderThread; |
| 616 | readonly taskIds: ReadonlyArray<string>; |
| 617 | readonly awaited: boolean; |
| 618 | }) => |
| 619 | Effect.sync(() => { |
| 620 | calls.push({ |
| 621 | providerThreadId: input.providerThread.id, |
| 622 | awaited: input.awaited, |
| 623 | }); |
| 624 | return { matched: input.taskIds.filter((taskId) => taskId === "tests") }; |
| 625 | }), |
| 626 | }), |
| 627 | } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime; |
| 628 | const layerTest = ThreadManagementService.layer.pipe( |
| 629 | Layer.provide( |
| 630 | Layer.mergeAll( |
| 631 | Layer.mock(Orchestrator.OrchestratorV2)({ |
| 632 | getThreadRecords: () => |
| 633 | Effect.succeed({ |
| 634 | thread: { |
| 635 | id: threadId, |
| 636 | deletedAt: null, |
| 637 | activeProviderThreadId: providerThreadId, |
| 638 | }, |
| 639 | providerThreads: [providerThread], |
| 640 | } as unknown as OrchestrationV2ThreadProjection), |
| 641 | }), |
| 642 | Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ |
| 643 | get: () => |
| 644 | Effect.succeed(state === "no-session" ? Option.none() : Option.some(runtime)), |
| 645 | }), |
| 646 | ), |
| 647 | ), |
| 648 | Layer.provide(ServerSettings.layerTest()), |
| 649 | Layer.provide(layerReceiptsTest), |
| 650 | ); |
| 651 | const result = yield* Effect.gen(function* () { |
| 652 | const service = yield* ThreadManagementService.ThreadManagementService; |
| 653 | return yield* service |
| 654 | .awaitBackgroundTasks({ |
| 655 | threadId, |
| 656 | taskIds: state.startsWith("no-match") ? ["finished"] : ["tests", "finished"], |
| 657 | awaited: state !== "no-match-clear", |
| 658 | }) |
| 659 | .pipe(Effect.result); |
| 660 | }).pipe(Effect.provide(Layer.merge(layerTest, ThreadCommandExecutor.layer))); |
| 661 | |
| 662 | if (state === "live") { |
| 663 | expect(result).toMatchObject({ |
| 664 | success: { matched: ["tests"], unmatched: ["finished"] }, |
| 665 | }); |
| 666 | expect(calls).toEqual([{ providerThreadId, awaited: true }]); |
| 667 | } else if (state === "no-match") { |
| 668 | expect(result).toMatchObject({ |
| 669 | failure: { _tag: "ThreadManagementBackgroundWaitError", reason: "unmatched" }, |
| 670 | }); |
| 671 | expect(calls).toEqual([{ providerThreadId, awaited: true }]); |
| 672 | } else if (state === "no-match-clear") { |
| 673 | expect(result).toMatchObject({ success: { matched: [], unmatched: ["finished"] } }); |
| 674 | expect(calls).toEqual([{ providerThreadId, awaited: false }]); |
| 675 | } else { |
| 676 | expect(result).toMatchObject({ |
| 677 | failure: { |
| 678 | _tag: "ThreadManagementBackgroundWaitError", |
| 679 | reason: state === "no-session" ? "unavailable" : "unsupported", |
| 680 | }, |
| 681 | }); |
| 682 | expect(calls).toEqual([]); |
| 683 | } |
| 684 | }), |
| 685 | ); |
| 686 | |
| 687 | it.effect.each([ |
| 688 | { status: "completed" as const, followUp: "none", settles: true }, |
| 689 | { status: "failed" as const, followUp: "none", settles: false }, |
| 690 | { status: "interrupted" as const, followUp: "none", settles: false }, |
| 691 | { status: "completed" as const, followUp: "notification", settles: true }, |
| 692 | { status: "completed" as const, followUp: "replayed-notification", settles: true }, |
| 693 | { status: "completed" as const, followUp: "delegated", settles: true }, |
| 694 | { status: "completed" as const, followUp: "failed-notification", settles: false }, |
| 695 | { status: "completed" as const, followUp: "held-notification", settles: true }, |
| 696 | { status: "completed" as const, followUp: "user", settles: false }, |
| 697 | { status: "completed" as const, followUp: "steered-notification", settles: false }, |
| 698 | // Send now cancels the queued run; its message joins the run that asked to settle. |
| 699 | { status: "completed" as const, followUp: "promoted-to-steer", settles: true }, |
| 700 | // A delegated-completion wake cancelled unstarted because the turn already took the result. |
| 701 | { status: "completed" as const, followUp: "cancelled-delegated", settles: true }, |
| 702 | { status: "completed" as const, followUp: "stopped-delegated", settles: false }, |
| 703 | ])("settleAfterRun handles $status with $followUp follow-up", ({ status, followUp, settles }) => |
| 704 | Effect.gen(function* () { |
| 705 | const projectId = ProjectId.make("project:thread-management:settle-after-run"); |
| 706 | const threadId = ThreadId.make("thread:thread-management:settle-after-run"); |
| 707 | const runId = RunId.make("run:thread-management:settle-after-run"); |
| 708 | const followUpId = RunId.make(`${runId}:follow-up`); |
| 709 | const subscribed = yield* Deferred.make<void>(); |
| 710 | const events = yield* Queue.unbounded<OrchestrationV2StoredEvent>(); |
| 711 | const replayRejected = yield* Deferred.make<void>(); |
| 712 | let sequence = 0; |
| 713 | let sequenceReads = 0; |
| 714 | let rejectedCommandId: CommandId | undefined; |
| 715 | const runs = [{ id: runId, ordinal: 1, status, userMessageId: MessageId.make("original") }]; |
| 716 | const nextRun = { |
| 717 | id: followUpId, |
| 718 | ordinal: 2, |
| 719 | status: "running" as OrchestrationV2Run["status"], |
| 720 | userMessageId: MessageId.make("follow-up"), |
| 721 | startedAt: followUp === "stopped-delegated" ? "2026-01-01T00:00:00.000Z" : null, |
| 722 | }; |
| 723 | const dispatched: Array<string> = []; |
| 724 | // The notification starts after the completed run was observed, before settlement commits. |
| 725 | let raced = false; |
| 726 | const layerTest = ThreadManagementService.layer.pipe( |
| 727 | Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), |
| 728 | Layer.provide( |
| 729 | Layer.mock(Orchestrator.OrchestratorV2)({ |
| 730 | getThreadEventSequence: () => |
| 731 | Effect.sync(() => { |
| 732 | if (followUp === "replayed-notification" && sequenceReads++ === 2) sequence = 1; |
| 733 | return sequence; |
| 734 | }), |
| 735 | restartContinuationPending: () => Effect.succeed(false), |
| 736 | streamStoredEventsFrom: () => |
| 737 | Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe( |
| 738 | Stream.drain, |
| 739 | Stream.concat(Stream.fromQueue(events)), |
| 740 | ), |
| 741 | getThreadRecords: (_threadId, _fields, filter) => |
| 742 | Effect.sync( |
| 743 | () => |
| 744 | ({ |
| 745 | thread: { id: threadId, projectId, deletedAt: null }, |
| 746 | runs: raced ? [...runs, nextRun] : runs, |
| 747 | messages: [ |
| 748 | { id: MessageId.make("original"), runId, role: "user" as const }, |
| 749 | { |
| 750 | id: nextRun.userMessageId, |
| 751 | runId: followUp === "promoted-to-steer" ? runId : followUpId, |
| 752 | role: "user" as const, |
| 753 | ...(followUp.endsWith("notification") ? { notification: {} } : {}), |
| 754 | ...(followUp.endsWith("delegated") ? { delegatedCompletion: {} } : {}), |
| 755 | }, |
| 756 | { id: MessageId.make("reply"), runId: followUpId, role: "assistant" as const }, |
| 757 | ...(followUp === "steered-notification" |
| 758 | ? [ |
| 759 | { |
| 760 | id: MessageId.make("user-steer"), |
| 761 | runId: followUpId, |
| 762 | role: "user" as const, |
| 763 | }, |
| 764 | ] |
| 765 | : []), |
| 766 | ].filter( |
| 767 | (message) => |
| 768 | (filter?.messageIds === undefined || |
| 769 | filter.messageIds.includes(message.id)) && |
| 770 | (filter?.messageRunIds === undefined || |
| 771 | filter.messageRunIds.includes(message.runId)) && |
| 772 | (filter?.messageRoles === undefined || |
| 773 | filter.messageRoles.includes(message.role)), |
| 774 | ), |
| 775 | }) as unknown as OrchestrationV2ThreadProjection, |
| 776 | ), |
| 777 | dispatch: (command) => |
| 778 | Effect.gen(function* () { |
| 779 | if (followUp !== "none" && !raced) { |
| 780 | raced = true; |
| 781 | rejectedCommandId = command.commandId; |
| 782 | return yield* new Orchestrator.OrchestratorDispatchError({ |
| 783 | commandId: command.commandId, |
| 784 | commandType: command.type, |
| 785 | cause: "Thread is still running", |
| 786 | }); |
| 787 | } |
| 788 | if (followUp === "replayed-notification" && command.commandId === rejectedCommandId) { |
| 789 | yield* Deferred.succeed(replayRejected, undefined); |
| 790 | return yield* new Orchestrator.OrchestratorCommandPreviouslyRejectedError({ |
| 791 | commandId: command.commandId, |
| 792 | commandType: command.type, |
| 793 | detail: "Thread is still running", |
| 794 | }); |
| 795 | } |
| 796 | dispatched.push(command.type); |
| 797 | return { sequence: 1, storedEvents: [] } as never; |
| 798 | }), |
| 799 | }), |
| 800 | ), |
| 801 | Layer.provide(ServerSettings.layerTest()), |
| 802 | Layer.provide(layerReceiptsTest), |
| 803 | ); |
| 804 | const service = yield* ThreadManagementService.ThreadManagementService.pipe( |
| 805 | Effect.provide(layerTest), |
| 806 | ); |
| 807 | const fiber = yield* service |
| 808 | .settleAfterRun({ projectId, threadId, runId }) |
| 809 | .pipe(Effect.forkChild); |
| 810 | if (followUp !== "none") { |
| 811 | yield* Deferred.await(subscribed); |
| 812 | expect(dispatched).toEqual([]); |
| 813 | nextRun.status = |
| 814 | followUp === "failed-notification" |
| 815 | ? "failed" |
| 816 | : followUp === "held-notification" |
| 817 | ? "queued" |
| 818 | : followUp === "promoted-to-steer" || followUp.endsWith("ed-delegated") |
| 819 | ? "cancelled" |
| 820 | : "completed"; |
| 821 | yield* Queue.offer(events, { |
| 822 | sequence: 1, |
| 823 | event: { type: "run.updated", threadId, payload: nextRun }, |
| 824 | } as unknown as OrchestrationV2StoredEvent); |
| 825 | } |
| 826 | if (followUp === "replayed-notification") { |
| 827 | yield* Deferred.await(replayRejected); |
| 828 | expect(dispatched).toEqual([]); |
| 829 | sequence = 2; |
| 830 | yield* Queue.offer(events, { |
| 831 | sequence, |
| 832 | event: { type: "run.updated", threadId, payload: nextRun }, |
| 833 | } as unknown as OrchestrationV2StoredEvent); |
| 834 | } |
| 835 | yield* Fiber.join(fiber); |
| 836 | expect(dispatched).toEqual(settles ? ["thread.settle"] : []); |
| 837 | }), |
| 838 | ); |
| 839 | |
| 840 | it.effect.each([ |
| 841 | { |
| 842 | name: "kills the thread's own agent and ends its turn", |
| 843 | others: [] as ReadonlyArray<string>, |
| 844 | killed: true, |
| 845 | expected: { outcome: "killed", sharedWithThreadIds: [] }, |
| 846 | teardown: ["release"], |
| 847 | }, |
| 848 | { |
| 849 | name: "keeps a process another working thread shares and only ends this turn", |
| 850 | others: ["thread:thread-management:force-stop:other"], |
| 851 | killed: false, |
| 852 | expected: { |
| 853 | outcome: "shared", |
| 854 | sharedWithThreadIds: ["thread:thread-management:force-stop:other"], |
| 855 | }, |
| 856 | teardown: ["detach"], |
| 857 | }, |
| 858 | { |
| 859 | name: "still settles a finished run's background work in a shared process", |
| 860 | others: ["thread:thread-management:force-stop:other"], |
| 861 | killed: false, |
| 862 | settledRun: true, |
| 863 | expected: { |
| 864 | outcome: "shared", |
| 865 | sharedWithThreadIds: ["thread:thread-management:force-stop:other"], |
| 866 | }, |
| 867 | teardown: ["detach"], |
| 868 | }, |
| 869 | { |
| 870 | name: "detaches when a sibling starts a turn just before the session is released", |
| 871 | others: [] as ReadonlyArray<string>, |
| 872 | killed: false, |
| 873 | siblingBusy: true, |
| 874 | expected: { outcome: "shared", sharedWithThreadIds: [] }, |
| 875 | teardown: ["release", "detach"], |
| 876 | }, |
| 877 | { |
| 878 | name: "reports a teardown that has not finished in time as still releasing", |
| 879 | others: [] as ReadonlyArray<string>, |
| 880 | killed: false, |
| 881 | releaseHangs: true, |
| 882 | expected: { outcome: "releasing", sharedWithThreadIds: [] }, |
| 883 | teardown: ["release"], |
| 884 | }, |
| 885 | ])("Force stop $name", ({ others, killed, expected, teardown, ...row }) => |
| 886 | Effect.gen(function* () { |
| 887 | const threadId = ThreadId.make("thread:thread-management:force-stop"); |
| 888 | const providerSessionId = ProviderSessionId.make("provider-session:force-stop"); |
| 889 | const providerThreadId = ProviderThreadId.make("provider-thread:force-stop"); |
| 890 | const runId = RunId.make("run:thread-management:force-stop"); |
| 891 | const dispatched: Array<OrchestrationV2Command | { readonly type: string }> = []; |
| 892 | const teardowns: Array<string> = []; |
| 893 | const tornDown = yield* Deferred.make<void>(); |
| 894 | const layerTest = ThreadManagementService.layer.pipe( |
| 895 | Layer.provide( |
| 896 | Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ |
| 897 | isLive: () => Effect.succeed(true), |
| 898 | otherWorkingThreads: () => Effect.succeed(others.map((id) => ThreadId.make(id))), |
| 899 | releaseUnlessBusy: () => |
| 900 | Effect.sync(() => teardowns.push("release")).pipe( |
| 901 | Effect.andThen( |
| 902 | "releaseHangs" in row ? Effect.never : Effect.succeed(!("siblingBusy" in row)), |
| 903 | ), |
| 904 | ), |
| 905 | detach: () => |
| 906 | Effect.sync(() => teardowns.push("detach")).pipe( |
| 907 | Effect.andThen(Deferred.succeed(tornDown, undefined)), |
| 908 | Effect.asVoid, |
| 909 | ), |
| 910 | }), |
| 911 | ), |
| 912 | Layer.provide( |
| 913 | Layer.mock(Orchestrator.OrchestratorV2)({ |
| 914 | getThreadRecords: () => |
| 915 | Effect.succeed({ |
| 916 | thread: { id: threadId, deletedAt: null }, |
| 917 | runs: [ |
| 918 | { |
| 919 | id: runId, |
| 920 | ordinal: 1, |
| 921 | status: "settledRun" in row ? "completed" : "running", |
| 922 | activeAttemptId: "attempt", |
| 923 | }, |
| 924 | // A queued follow-up neither supersedes the request nor becomes the target. |
| 925 | { id: RunId.make("run:queued-follow-up"), ordinal: 2, status: "queued" }, |
| 926 | ], |
| 927 | providerThreads: [{ id: providerThreadId, appThreadId: threadId, providerSessionId }], |
| 928 | providerTurns: [ |
| 929 | { |
| 930 | id: "provider-turn:force-stop", |
| 931 | providerThreadId, |
| 932 | runAttemptId: "attempt", |
| 933 | status: "settledRun" in row ? "completed" : "running", |
| 934 | }, |
| 935 | ], |
| 936 | } as unknown as OrchestrationV2ThreadProjection), |
| 937 | dispatch: (command) => |
| 938 | Effect.sync(() => { |
| 939 | dispatched.push(command); |
| 940 | return { sequence: 1, storedEvents: [] } as never; |
| 941 | }), |
| 942 | }), |
| 943 | ), |
| 944 | Layer.provide( |
| 945 | Layer.succeed(AgentScope, { |
| 946 | ...AgentScope.defaultValue(), |
| 947 | kill: () => Effect.succeed(killed), |
| 948 | }), |
| 949 | ), |
| 950 | Layer.provide(ServerSettings.layerTest()), |
| 951 | Layer.provide(layerReceiptsTest), |
| 952 | ); |
| 953 | yield* Effect.gen(function* () { |
| 954 | const service = yield* ThreadManagementService.ThreadManagementService; |
| 955 | const stopping = yield* service.forceStopThread({ threadId, runId }).pipe(Effect.forkChild); |
| 956 | yield* TestClock.adjust("10 seconds"); |
| 957 | expect(yield* Fiber.join(stopping)).toEqual(expected); |
| 958 | expect(dispatched).toMatchObject([ |
| 959 | { |
| 960 | type: "thread.background-work.settle", |
| 961 | providerThreadId, |
| 962 | reason: "Force stopped by user", |
| 963 | }, |
| 964 | ]); |
| 965 | if (teardown.at(-1) === "detach") yield* Deferred.await(tornDown); |
| 966 | expect(teardowns).toEqual(teardown); |
| 967 | }).pipe(Effect.provide(layerTest)); |
| 968 | }), |
| 969 | ); |
| 970 | |
| 971 | it.effect("Force stop refuses once a newer run has started, before killing anything", () => |
| 972 | Effect.gen(function* () { |
| 973 | const threadId = ThreadId.make("thread:thread-management:force-stop-superseded"); |
| 974 | let killed = false; |
| 975 | const layerTest = ThreadManagementService.layer.pipe( |
| 976 | Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), |
| 977 | Layer.provide( |
| 978 | Layer.mock(Orchestrator.OrchestratorV2)({ |
| 979 | getThreadRecords: () => |
| 980 | Effect.succeed({ |
| 981 | thread: { id: threadId, deletedAt: null }, |
| 982 | runs: [ |
| 983 | { id: RunId.make("run:stopped"), ordinal: 1, status: "running" }, |
| 984 | { id: RunId.make("run:newer"), ordinal: 2, status: "running" }, |
| 985 | ], |
| 986 | providerThreads: [], |
| 987 | providerTurns: [], |
| 988 | } as unknown as OrchestrationV2ThreadProjection), |
| 989 | }), |
| 990 | ), |
| 991 | Layer.provide( |
| 992 | Layer.succeed(AgentScope, { |
| 993 | ...AgentScope.defaultValue(), |
| 994 | kill: () => Effect.sync(() => (killed = true)), |
| 995 | }), |
| 996 | ), |
| 997 | Layer.provide(ServerSettings.layerTest()), |
| 998 | Layer.provide(layerReceiptsTest), |
| 999 | ); |
| 1000 | const error = yield* Effect.gen(function* () { |
| 1001 | const service = yield* ThreadManagementService.ThreadManagementService; |
| 1002 | return yield* Effect.flip( |
| 1003 | service.forceStopThread({ threadId, runId: RunId.make("run:stopped") }), |
| 1004 | ); |
| 1005 | }).pipe(Effect.provide(layerTest)); |
| 1006 | expect(error.reason).toBe("superseded"); |
| 1007 | expect(killed).toBe(false); |
| 1008 | }), |
| 1009 | ); |
| 1010 | |
| 1011 | it.effect( |
| 1012 | "admits two transcript exports at a time and frees a slot on failure or interruption", |
| 1013 | () => |
| 1014 | Effect.gen(function* () { |
| 1015 | const started = yield* Queue.unbounded<string>(); |
| 1016 | const gates = new Map( |
| 1017 | ["a", "b", "c", "d", "e", "f"].map((id) => [ |
| 1018 | id, |
| 1019 | Deferred.makeUnsafe<void, Orchestrator.OrchestratorV2Error>(), |
| 1020 | ]), |
| 1021 | ); |
| 1022 | const layerTest = ThreadManagementService.layer.pipe( |
| 1023 | Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), |
| 1024 | Layer.provide( |
| 1025 | Layer.mock(Orchestrator.OrchestratorV2)({ |
| 1026 | getTranscriptByteFloor: () => Effect.succeed(0), |
| 1027 | getThreadSnapshot: (threadId) => |
| 1028 | Queue.offer(started, threadId).pipe( |
| 1029 | Effect.andThen(Deferred.await(gates.get(threadId)!)), |
| 1030 | Effect.as({ |
| 1031 | schemaVersion: 1, |
| 1032 | snapshotSequence: 1, |
| 1033 | projection: { |
| 1034 | thread: { id: threadId, title: threadId }, |
| 1035 | updatedAt: DateTime.makeUnsafe(0), |
| 1036 | visibleTurnItems: [], |
| 1037 | } as unknown as OrchestrationV2ThreadProjection, |
| 1038 | }), |
| 1039 | ), |
| 1040 | }), |
| 1041 | ), |
| 1042 | Layer.provide(ServerSettings.layerTest()), |
| 1043 | Layer.provide(layerReceiptsTest), |
| 1044 | ); |
| 1045 | |
| 1046 | yield* Effect.gen(function* () { |
| 1047 | const threads = yield* ThreadManagementService.ThreadManagementService; |
| 1048 | const exportThread = (id: string) => |
| 1049 | threads.getThreadTranscript(ThreadId.make(id)).pipe( |
| 1050 | Effect.provideService(ServerEnvironment.ServerEnvironmentIdentity, { |
| 1051 | getEnvironmentId: Effect.succeed(EnvironmentId.make("environment:export")), |
| 1052 | }), |
| 1053 | Effect.forkChild({ startImmediately: true }), |
| 1054 | ); |
| 1055 | const a = yield* exportThread("a"); |
| 1056 | const b = yield* exportThread("b"); |
| 1057 | const c = yield* exportThread("c"); |
| 1058 | const d = yield* exportThread("d"); |
| 1059 | const e = yield* exportThread("e"); |
| 1060 | const f = yield* exportThread("f"); |
| 1061 | expect(yield* Queue.takeAll(started)).toEqual(["a", "b"]); |
| 1062 | |
| 1063 | yield* Fiber.interrupt(c); |
| 1064 | yield* Deferred.fail( |
| 1065 | gates.get("a")!, |
| 1066 | new Orchestrator.OrchestratorProjectionError({ threadId: ThreadId.make("a"), cause: "" }), |
| 1067 | ); |
| 1068 | expect(yield* Queue.take(started)).toBe("d"); |
| 1069 | yield* Fiber.interrupt(b); |
| 1070 | expect(yield* Queue.take(started)).toBe("e"); |
| 1071 | yield* Deferred.succeed(gates.get("d")!, undefined); |
| 1072 | expect(yield* Queue.take(started)).toBe("f"); |
| 1073 | yield* Deferred.succeed(gates.get("e")!, undefined); |
| 1074 | yield* Deferred.succeed(gates.get("f")!, undefined); |
| 1075 | |
| 1076 | expect((yield* Fiber.await(a))._tag).toBe("Failure"); |
| 1077 | expect((yield* Fiber.join(d)).threadId).toBe("d"); |
| 1078 | expect((yield* Fiber.join(e)).threadId).toBe("e"); |
| 1079 | expect((yield* Fiber.join(f)).threadId).toBe("f"); |
| 1080 | expect(yield* Queue.size(started)).toBe(0); |
| 1081 | }).pipe(Effect.provide(layerTest)); |
| 1082 | }), |
| 1083 | ); |