The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; |
| 2 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 3 | import { assert, it } from "@effect/vitest"; |
| 4 | import { |
| 5 | CommandId, |
| 6 | EnvironmentId, |
| 7 | EventId, |
| 8 | MessageId, |
| 9 | NodeId, |
| 10 | ProjectId, |
| 11 | ProviderDriverKind, |
| 12 | ProviderInstanceId, |
| 13 | ProviderThreadId, |
| 14 | ProviderTurnId, |
| 15 | RunId, |
| 16 | RuntimeRequestId, |
| 17 | ThreadId, |
| 18 | TurnItemId, |
| 19 | type OrchestrationV2DomainEvent, |
| 20 | type ServerProvider, |
| 21 | } from "@t3tools/contracts"; |
| 22 | import * as DateTime from "effect/DateTime"; |
| 23 | import * as Effect from "effect/Effect"; |
| 24 | import * as Fiber from "effect/Fiber"; |
| 25 | import * as FileSystem from "effect/FileSystem"; |
| 26 | import * as Layer from "effect/Layer"; |
| 27 | import * as Path from "effect/Path"; |
| 28 | import * as Queue from "effect/Queue"; |
| 29 | import * as Stream from "effect/Stream"; |
| 30 | |
| 31 | import { ClaudeProviderCapabilitiesV2 } from "../orchestration-v2/Adapters/ClaudeAdapterV2.ts"; |
| 32 | import * as CommandReceiptStore from "../orchestration-v2/CommandReceiptStore.ts"; |
| 33 | import * as EffectWorker from "../orchestration-v2/EffectWorker.ts"; |
| 34 | import * as EventSink from "../orchestration-v2/EventSink.ts"; |
| 35 | import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; |
| 36 | import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; |
| 37 | import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts"; |
| 38 | import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; |
| 39 | import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; |
| 40 | import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; |
| 41 | import * as ServerSettings from "../serverSettings.ts"; |
| 42 | import * as ProviderReplayHarness from "../orchestration-v2/testkit/ProviderReplayHarness.ts"; |
| 43 | import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; |
| 44 | import * as SqlitePersistence from "../persistence/Sqlite.ts"; |
| 45 | import * as ProjectService from "../project/ProjectService.ts"; |
| 46 | import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; |
| 47 | import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; |
| 48 | import * as SecretRequests from "../secrets/SecretRequests.ts"; |
| 49 | import type { McpInvocationScope } from "./McpInvocationContext.ts"; |
| 50 | import * as OrchestratorMcpService from "./OrchestratorMcpService.ts"; |
| 51 | import * as RemoteDelegation from "../peer/RemoteDelegation.ts"; |
| 52 | |
| 53 | const layerOrchestratorService = OrchestratorMcpService.layer.pipe( |
| 54 | Layer.provide(Layer.mock(ThreadLaunchService.ThreadLaunchService)({})), |
| 55 | Layer.provide(Layer.mock(GitVcsDriver.GitVcsDriver)({})), |
| 56 | ); |
| 57 | |
| 58 | const driver = ProviderDriverKind.make("claudeAgent"); |
| 59 | const instanceId = ProviderInstanceId.make("claudeAgent"); |
| 60 | const modelSelection = { instanceId, model: "claude-test" }; |
| 61 | const provider = { |
| 62 | instanceId, |
| 63 | driver, |
| 64 | enabled: true, |
| 65 | installed: true, |
| 66 | status: "ready", |
| 67 | auth: { status: "authenticated" }, |
| 68 | models: [{ slug: modelSelection.model, name: modelSelection.model, isCustom: false }], |
| 69 | } as unknown as ServerProvider; |
| 70 | |
| 71 | // Cancellation races with turn completion and checkpoint capture; "recorded" restarts before the Stop outbox runs. |
| 72 | const cancelBackgroundOnlyTask = ( |
| 73 | background: "subagent" | "command", |
| 74 | { |
| 75 | cancelAt = "settled", |
| 76 | remote = false, |
| 77 | }: { |
| 78 | readonly remote?: boolean; |
| 79 | readonly cancelAt?: |
| 80 | | "settled" |
| 81 | | "checkpoint" |
| 82 | | "turn" |
| 83 | | "recorded" |
| 84 | | "monitor" |
| 85 | | "refused" |
| 86 | | "delegate"; |
| 87 | } = {}, |
| 88 | ) => |
| 89 | Effect.scoped( |
| 90 | Effect.gen(function* () { |
| 91 | const cwd = yield* checkpointWorkspace(`task-cancel-background-${background}`); |
| 92 | const turns: Array<{ |
| 93 | readonly turn: ProviderAdapter.ProviderAdapterV2TurnInput; |
| 94 | readonly events: Queue.Queue<ProviderAdapter.ProviderAdapterV2Event>; |
| 95 | }> = []; |
| 96 | const interrupted: Array<ReadonlyArray<string> | undefined> = []; |
| 97 | const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { |
| 98 | instanceId, |
| 99 | driver, |
| 100 | getCapabilities: () => Effect.succeed(ClaudeProviderCapabilitiesV2), |
| 101 | planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), |
| 102 | openSession: (input) => |
| 103 | Effect.gen(function* () { |
| 104 | const now = yield* DateTime.now; |
| 105 | const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>(); |
| 106 | return { |
| 107 | instanceId, |
| 108 | driver, |
| 109 | providerSessionId: input.providerSessionId, |
| 110 | providerSession: { |
| 111 | id: input.providerSessionId, |
| 112 | driver, |
| 113 | providerInstanceId: instanceId, |
| 114 | status: "ready", |
| 115 | cwd, |
| 116 | model: modelSelection.model, |
| 117 | capabilities: ClaudeProviderCapabilitiesV2, |
| 118 | createdAt: now, |
| 119 | updatedAt: now, |
| 120 | lastError: null, |
| 121 | }, |
| 122 | events: Stream.fromQueue(events), |
| 123 | ensureThread: ({ threadId }) => |
| 124 | Effect.succeed({ |
| 125 | id: ProviderThreadId.make(`provider-thread:${threadId}`), |
| 126 | driver, |
| 127 | providerInstanceId: instanceId, |
| 128 | providerSessionId: input.providerSessionId, |
| 129 | appThreadId: threadId, |
| 130 | ownerNodeId: null, |
| 131 | nativeThreadRef: { driver, nativeId: `native:${threadId}`, strength: "strong" }, |
| 132 | nativeConversationHeadRef: null, |
| 133 | status: "idle", |
| 134 | firstRunOrdinal: null, |
| 135 | lastRunOrdinal: null, |
| 136 | handoffIds: [], |
| 137 | forkedFrom: null, |
| 138 | createdAt: now, |
| 139 | updatedAt: now, |
| 140 | }), |
| 141 | resumeThread: ({ providerThread }) => Effect.succeed(providerThread), |
| 142 | startTurn: (turn) => |
| 143 | Effect.gen(function* () { |
| 144 | turns.push({ turn, events }); |
| 145 | yield* Queue.offer(events, { |
| 146 | type: "provider_turn.updated", |
| 147 | driver, |
| 148 | providerTurn: { |
| 149 | id: ProviderTurnId.make(`provider-turn:${turn.attemptId}`), |
| 150 | providerThreadId: turn.providerThread.id, |
| 151 | nodeId: turn.rootNodeId, |
| 152 | runAttemptId: turn.attemptId, |
| 153 | nativeTurnRef: null, |
| 154 | ordinal: turn.providerTurnOrdinal, |
| 155 | status: "running", |
| 156 | startedAt: now, |
| 157 | completedAt: null, |
| 158 | }, |
| 159 | }); |
| 160 | }), |
| 161 | steerTurn: () => Effect.die("unused"), |
| 162 | interruptTurn: ({ providerThread }) => |
| 163 | Effect.sync(() => { |
| 164 | interrupted.push(providerThread.pendingBackgroundTasks?.map((t) => t.taskId)); |
| 165 | }), |
| 166 | respondToRuntimeRequest: () => Effect.die("unused"), |
| 167 | readThreadSnapshot: () => Effect.die("unused"), |
| 168 | rollbackThread: () => Effect.die("unused"), |
| 169 | forkThread: () => Effect.die("unused"), |
| 170 | }; |
| 171 | }), |
| 172 | }; |
| 173 | const layerAdapters = ProviderAdapterRegistry.layerSingle(adapter); |
| 174 | const fs = yield* FileSystem.FileSystem; |
| 175 | const path = yield* Path.Path; |
| 176 | const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-task-cancel-" }); |
| 177 | const databaseLayer = SqlitePersistence.layerFromPath( |
| 178 | path.join(directory, "state.sqlite"), |
| 179 | ).pipe(Layer.provide(NodeServices.layer)); |
| 180 | // Migrate the file first, as an existing install's database already is. |
| 181 | yield* Effect.provide(Effect.void, databaseLayer); |
| 182 | // Each build is a server on the same database; the second runs startup recovery. |
| 183 | let restarted = false; |
| 184 | const layerOrchestrator = Layer.suspend(() => |
| 185 | ProviderReplayHarness.layerWithRegistry( |
| 186 | { name: "task-cancel-native-subagent" }, |
| 187 | layerAdapters, |
| 188 | restarted |
| 189 | ? { databaseLayer, recoverOnStartup: true } |
| 190 | : { databaseLayer, runEffectWorker: false }, |
| 191 | ), |
| 192 | ); |
| 193 | const layerOrchestration = Layer.merge( |
| 194 | layerOrchestrator, |
| 195 | Layer.effect( |
| 196 | ThreadManagementService.ThreadManagementService, |
| 197 | Effect.map(ThreadManagementService.ThreadManagementService, (service) => |
| 198 | cancelAt === "refused" |
| 199 | ? { |
| 200 | ...service, |
| 201 | stopDelegatedTasks: () => |
| 202 | Effect.fail( |
| 203 | new Orchestrator.OrchestratorDispatchError({ |
| 204 | commandId: CommandId.make("refused-descendant-stop"), |
| 205 | commandType: "thread.stop", |
| 206 | cause: "descendant refused stop", |
| 207 | }), |
| 208 | ), |
| 209 | } |
| 210 | : service, |
| 211 | ), |
| 212 | ).pipe( |
| 213 | Layer.provide( |
| 214 | ThreadManagementService.layer.pipe( |
| 215 | Layer.provide(layerOrchestrator), |
| 216 | Layer.provide(ServerSettings.layerTest()), |
| 217 | Layer.provide( |
| 218 | CommandReceiptStore.layer.pipe(Layer.provide(SqlitePersistence.layerMemory)), |
| 219 | ), |
| 220 | ), |
| 221 | ), |
| 222 | Layer.provide(layerOrchestrator), |
| 223 | Layer.provide(ServerSettings.layerTest()), |
| 224 | Layer.provide( |
| 225 | CommandReceiptStore.layer.pipe(Layer.provide(SqlitePersistence.layerMemory)), |
| 226 | ), |
| 227 | ), |
| 228 | ); |
| 229 | const layerTest = layerOrchestratorService.pipe( |
| 230 | Layer.provideMerge(layerOrchestration), |
| 231 | Layer.provide( |
| 232 | Layer.mergeAll( |
| 233 | layerAdapters, |
| 234 | Layer.mock(ProviderRegistry.ProviderRegistry)({ |
| 235 | getProviders: Effect.succeed([provider]), |
| 236 | }), |
| 237 | Layer.mock(ProjectService.ProjectService)({}), |
| 238 | Layer.mock(SecretRequests.SecretRequests)({}), |
| 239 | Layer.mock(RemoteDelegation.RemoteDelegation)({}), |
| 240 | ServerSettings.layerTest(), |
| 241 | Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), |
| 242 | NodeServices.layer, |
| 243 | NodeCrypto.layer, |
| 244 | ), |
| 245 | ), |
| 246 | ); |
| 247 | |
| 248 | const stopped = yield* Effect.gen(function* () { |
| 249 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 250 | const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; |
| 251 | const service = yield* OrchestratorMcpService.OrchestratorMcpService; |
| 252 | const eventSink = yield* EventSink.EventSinkV2; |
| 253 | const parentThreadId = ThreadId.make("thread:task-cancel-parent"); |
| 254 | // Read the cursor before forking: provider events offered right after a watch can |
| 255 | // commit before a live-tail subscriber has read its starting sequence. |
| 256 | const watch = (predicate: (event: OrchestrationV2DomainEvent) => boolean) => |
| 257 | eventSink.latestSequence().pipe( |
| 258 | Effect.flatMap((afterSequence) => |
| 259 | eventSink.stream({ afterSequence }).pipe( |
| 260 | Stream.map((stored) => stored.event), |
| 261 | Stream.filter(predicate), |
| 262 | Stream.take(1), |
| 263 | Stream.runDrain, |
| 264 | Effect.forkScoped, |
| 265 | ), |
| 266 | ), |
| 267 | ); |
| 268 | const turnRunning = () => |
| 269 | watch( |
| 270 | (event) => event.type === "provider-turn.updated" && event.payload.status === "running", |
| 271 | ); |
| 272 | yield* orchestrator.dispatch({ |
| 273 | type: "thread.create", |
| 274 | commandId: CommandId.make("create-parent"), |
| 275 | threadId: parentThreadId, |
| 276 | projectId: ProjectId.make("project:task-cancel"), |
| 277 | title: "Parent", |
| 278 | modelSelection, |
| 279 | runtimeMode: "full-access", |
| 280 | interactionMode: "default", |
| 281 | branch: null, |
| 282 | worktreePath: cwd, |
| 283 | createdBy: "user", |
| 284 | creationSource: "web", |
| 285 | }); |
| 286 | const parentRunning = yield* turnRunning(); |
| 287 | yield* orchestrator.dispatch({ |
| 288 | type: "message.dispatch", |
| 289 | commandId: CommandId.make("start-parent"), |
| 290 | threadId: parentThreadId, |
| 291 | messageId: MessageId.make("message:start-parent"), |
| 292 | text: "Delegate a review", |
| 293 | attachments: [], |
| 294 | dispatchMode: { type: "start_immediately" }, |
| 295 | createdBy: "user", |
| 296 | creationSource: "web", |
| 297 | }); |
| 298 | yield* worker.drain(); |
| 299 | yield* Fiber.join(parentRunning); |
| 300 | |
| 301 | const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs[0]!; |
| 302 | const childRunning = yield* turnRunning(); |
| 303 | yield* orchestrator.dispatch({ |
| 304 | type: "delegated_task.request", |
| 305 | commandId: CommandId.make("delegate"), |
| 306 | parentThreadId, |
| 307 | parentRunId: parentRun.id, |
| 308 | parentNodeId: parentRun.rootNodeId!, |
| 309 | task: "Review the change", |
| 310 | modelSelection, |
| 311 | runtimeMode: "full-access", |
| 312 | interactionMode: "default", |
| 313 | completionWake: "always", |
| 314 | createdBy: "agent", |
| 315 | creationSource: "mcp", |
| 316 | }); |
| 317 | if (cancelAt === "turn") { |
| 318 | const pendingTask = (yield* orchestrator.getThreadProjection(parentThreadId)) |
| 319 | .subagents[0]!; |
| 320 | const { runs } = yield* orchestrator.getThreadProjection(pendingTask.childThreadId!); |
| 321 | const now = yield* DateTime.now; |
| 322 | const current = runs[0]!; |
| 323 | const previous = { |
| 324 | ...current, |
| 325 | id: RunId.make("run:before-continuation"), |
| 326 | ordinal: 1, |
| 327 | status: "completed" as const, |
| 328 | startedAt: now, |
| 329 | completedAt: now, |
| 330 | }; |
| 331 | const at = { threadId: current.threadId, occurredAt: now }; |
| 332 | yield* eventSink.write({ |
| 333 | commandId: CommandId.make("command:runtime-reconcile:continuation"), |
| 334 | events: [ |
| 335 | { |
| 336 | ...at, |
| 337 | runId: current.id, |
| 338 | id: EventId.make("event:continuation"), |
| 339 | type: "run.updated", |
| 340 | payload: { ...current, ordinal: 2, restartContinuationOfRunId: previous.id }, |
| 341 | }, |
| 342 | { |
| 343 | ...at, |
| 344 | runId: previous.id, |
| 345 | id: EventId.make("event:previous"), |
| 346 | type: "run.updated", |
| 347 | payload: previous, |
| 348 | }, |
| 349 | ], |
| 350 | }); |
| 351 | } |
| 352 | yield* worker.drain(); |
| 353 | yield* Fiber.join(childRunning); |
| 354 | const task = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]!; |
| 355 | const child = turns.find((entry) => entry.turn.threadId === task.childThreadId)!; |
| 356 | const childTurn = (yield* orchestrator.getThreadProjection(task.childThreadId!)) |
| 357 | .providerTurns[0]!; |
| 358 | |
| 359 | const now = yield* DateTime.now; |
| 360 | const subagentId = NodeId.make("node:native-subagent"); |
| 361 | const native = { |
| 362 | threadId: task.childThreadId!, |
| 363 | runId: child.turn.runId, |
| 364 | origin: "provider_native", |
| 365 | driver, |
| 366 | providerInstanceId: instanceId, |
| 367 | providerThreadId: childTurn.providerThreadId, |
| 368 | childThreadId: null, |
| 369 | prompt: "Check the tests", |
| 370 | title: null, |
| 371 | status: "running", |
| 372 | result: null, |
| 373 | startedAt: now, |
| 374 | completedAt: null, |
| 375 | updatedAt: now, |
| 376 | } as const; |
| 377 | const waiting = yield* watch( |
| 378 | (event) => |
| 379 | event.type === "run.updated" && |
| 380 | event.payload.id === child.turn.runId && |
| 381 | event.payload.status === "waiting", |
| 382 | ); |
| 383 | const nativeSubagentEvents = [ |
| 384 | { |
| 385 | type: "node.updated", |
| 386 | driver, |
| 387 | node: { |
| 388 | id: subagentId, |
| 389 | threadId: native.threadId, |
| 390 | runId: native.runId, |
| 391 | parentNodeId: child.turn.rootNodeId, |
| 392 | rootNodeId: child.turn.rootNodeId, |
| 393 | kind: "subagent", |
| 394 | status: "running", |
| 395 | countsForRun: false, |
| 396 | providerThreadId: childTurn.providerThreadId, |
| 397 | providerTurnId: childTurn.id, |
| 398 | nativeItemRef: null, |
| 399 | runtimeRequestId: null, |
| 400 | checkpointScopeId: null, |
| 401 | startedAt: now, |
| 402 | completedAt: null, |
| 403 | }, |
| 404 | }, |
| 405 | { |
| 406 | type: "subagent.updated", |
| 407 | driver, |
| 408 | subagent: { |
| 409 | ...native, |
| 410 | id: subagentId, |
| 411 | parentNodeId: child.turn.rootNodeId, |
| 412 | createdBy: "agent", |
| 413 | nativeTaskRef: null, |
| 414 | model: null, |
| 415 | }, |
| 416 | }, |
| 417 | { |
| 418 | type: "turn_item.updated", |
| 419 | driver, |
| 420 | turnItem: { |
| 421 | ...native, |
| 422 | id: TurnItemId.make("turn-item:native-subagent"), |
| 423 | nodeId: subagentId, |
| 424 | providerTurnId: childTurn.id, |
| 425 | nativeItemRef: null, |
| 426 | parentItemId: null, |
| 427 | ordinal: 1, |
| 428 | type: "subagent", |
| 429 | subagentId, |
| 430 | }, |
| 431 | }, |
| 432 | ] satisfies ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>; |
| 433 | const shellEvents = [ |
| 434 | { |
| 435 | type: "provider_thread.updated", |
| 436 | driver, |
| 437 | providerThread: { |
| 438 | ...child.turn.providerThread, |
| 439 | status: "active", |
| 440 | pendingBackgroundTasks: [ |
| 441 | { taskId: "bg-sleep", description: "sleep 300", kind: "command", awaited: true }, |
| 442 | ], |
| 443 | updatedAt: now, |
| 444 | }, |
| 445 | }, |
| 446 | ] satisfies ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>; |
| 447 | const started = yield* watch((event) => |
| 448 | background === "subagent" |
| 449 | ? event.type === "subagent.updated" && event.payload.id === subagentId |
| 450 | : event.type === "provider-thread.updated" && |
| 451 | event.threadId === task.childThreadId && |
| 452 | (event.payload.pendingBackgroundTasks?.length ?? 0) > 0, |
| 453 | ); |
| 454 | // An async question the child asked during its turn, and one no run of this task owns. |
| 455 | const question = (id: string, nodeId: NodeId) => |
| 456 | ({ |
| 457 | type: "runtime_request.updated", |
| 458 | driver, |
| 459 | runtimeRequest: { |
| 460 | id: RuntimeRequestId.make(id), |
| 461 | nodeId, |
| 462 | providerTurnId: childTurn.id, |
| 463 | nativeRequestRef: null, |
| 464 | kind: "user_input", |
| 465 | status: "pending", |
| 466 | responseCapability: { type: "message" }, |
| 467 | createdAt: now, |
| 468 | resolvedAt: null, |
| 469 | }, |
| 470 | }) satisfies ProviderAdapter.ProviderAdapterV2Event; |
| 471 | for (const event of [ |
| 472 | question("request:owned", child.turn.rootNodeId), |
| 473 | question("request:unrelated", NodeId.make("node:unrelated")), |
| 474 | ...(background === "subagent" ? nativeSubagentEvents : shellEvents), |
| 475 | ]) { |
| 476 | yield* Queue.offer(child.events, event); |
| 477 | } |
| 478 | yield* Fiber.join(started); |
| 479 | if (remote) { |
| 480 | yield* orchestrator.dispatch({ |
| 481 | type: "delegated_task.remote.request", |
| 482 | commandId: CommandId.make("remote-descendant"), |
| 483 | parentThreadId: task.childThreadId!, |
| 484 | parentRunId: child.turn.runId, |
| 485 | parentNodeId: child.turn.rootNodeId, |
| 486 | task: "Work in a linked environment", |
| 487 | driver, |
| 488 | modelSelection, |
| 489 | remoteChild: { |
| 490 | environmentId: EnvironmentId.make("environment:remote-descendant"), |
| 491 | threadId: ThreadId.make("thread:remote-descendant"), |
| 492 | label: "Remote", |
| 493 | }, |
| 494 | }); |
| 495 | } |
| 496 | const turnEndEvents = [ |
| 497 | { |
| 498 | type: "provider_turn.updated", |
| 499 | driver, |
| 500 | providerTurn: { ...childTurn, status: "completed", completedAt: now }, |
| 501 | }, |
| 502 | { |
| 503 | type: "turn.terminal", |
| 504 | driver, |
| 505 | providerThreadId: childTurn.providerThreadId, |
| 506 | providerTurnId: childTurn.id, |
| 507 | runOrdinal: child.turn.runOrdinal, |
| 508 | status: "completed", |
| 509 | failure: null, |
| 510 | threadDisposition: "reusable", |
| 511 | }, |
| 512 | ] satisfies ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>; |
| 513 | const endTurn = Effect.gen(function* () { |
| 514 | for (const event of turnEndEvents) yield* Queue.offer(child.events, event); |
| 515 | yield* Fiber.join(waiting); |
| 516 | }); |
| 517 | if (cancelAt !== "turn") yield* endTurn; |
| 518 | const turnSettled = |
| 519 | cancelAt !== "checkpoint" && cancelAt !== "turn" && cancelAt !== "recorded"; |
| 520 | if (turnSettled) yield* worker.drain(); |
| 521 | if (cancelAt === "monitor") { |
| 522 | // A monitor notification then ran and completed while the shell kept running. |
| 523 | const { runs, messages } = yield* orchestrator.getThreadProjection(task.childThreadId!); |
| 524 | const run = { ...runs[0]!, id: RunId.make("run:monitor"), ordinal: 2 }; |
| 525 | const source = { kind: "monitor" } as const; |
| 526 | const notification = { source, outcome: "updated", summary: "Ran" } as const; |
| 527 | const user = { ...messages[0]!, id: MessageId.make("message:monitor"), runId: run.id }; |
| 528 | const message = { ...user, notification }; |
| 529 | const at = { threadId: run.threadId, runId: run.id, occurredAt: now }; |
| 530 | yield* (yield* EventSink.EventSinkV2).write({ |
| 531 | commandId: CommandId.make("command:runtime-reconcile:monitor"), |
| 532 | events: [ |
| 533 | { ...at, id: EventId.make("event:m1"), type: "message.updated", payload: message }, |
| 534 | { ...at, id: EventId.make("event:m2"), type: "run.updated", payload: run }, |
| 535 | ], |
| 536 | }); |
| 537 | } |
| 538 | |
| 539 | const scope: McpInvocationScope = { |
| 540 | environmentId: EnvironmentId.make("environment:task-cancel"), |
| 541 | requestNamespace: "provider-session:task-cancel", |
| 542 | thread: { |
| 543 | threadId: parentThreadId, |
| 544 | providerSessionId: "provider-session:task-cancel", |
| 545 | providerInstanceId: instanceId, |
| 546 | }, |
| 547 | client: undefined, |
| 548 | capabilities: new Set(["orchestration"]), |
| 549 | issuedAt: 1, |
| 550 | }; |
| 551 | if (cancelAt === "delegate") { |
| 552 | // The child's run settled while its native subagent still runs, and that subagent delegates. |
| 553 | const childScope: McpInvocationScope = { |
| 554 | ...scope, |
| 555 | thread: { ...scope.thread!, threadId: task.childThreadId! }, |
| 556 | }; |
| 557 | const delegated = yield* service.delegateTask(childScope, { task: "Reply OK." }); |
| 558 | assert.equal(delegated.status, "starting"); |
| 559 | return undefined; |
| 560 | } |
| 561 | const before = yield* service.taskStatus(scope, task.id); |
| 562 | assert.equal(before.status, "running"); |
| 563 | assert.equal(before.workState, turnSettled ? "waiting_for_children" : "working"); |
| 564 | |
| 565 | const parentTask = Effect.map(orchestrator.getThreadProjection(parentThreadId), (parent) => |
| 566 | parent.subagents.find((entry) => entry.id === task.id), |
| 567 | ); |
| 568 | if (cancelAt === "recorded") { |
| 569 | const commandId = CommandId.make("stop-recorded"); |
| 570 | yield* orchestrator.cancelDelegatedTask({ |
| 571 | parentThreadId, |
| 572 | taskId: task.id, |
| 573 | command: { |
| 574 | type: "thread.stop", |
| 575 | commandId, |
| 576 | threadId: task.childThreadId!, |
| 577 | stopDelegatedTasks: false, |
| 578 | }, |
| 579 | stopDescendants: () => Effect.void, |
| 580 | }); |
| 581 | const committed = yield* eventSink.readByCommandId({ commandId }).pipe(Stream.runCollect); |
| 582 | assert.isTrue( |
| 583 | committed.some( |
| 584 | ({ event }) => |
| 585 | event.type === "subagent.updated" && event.payload.cancelledRunId !== undefined, |
| 586 | ), |
| 587 | ); |
| 588 | assert.isTrue( |
| 589 | committed.some( |
| 590 | ({ event }) => |
| 591 | event.type === "turn-item.updated" && |
| 592 | event.payload.type === "run_interrupt_request", |
| 593 | ), |
| 594 | ); |
| 595 | assert.equal((yield* parentTask)?.status, "running"); |
| 596 | return { scope, taskId: task.id, settled: undefined }; |
| 597 | } |
| 598 | if (cancelAt === "refused") { |
| 599 | yield* Effect.flip( |
| 600 | service.cancelTask(scope, { taskId: task.id, clientRequestId: "refused-cancel" }), |
| 601 | ); |
| 602 | const after = yield* parentTask; |
| 603 | assert.equal(after?.status, "running"); |
| 604 | assert.equal(after?.result, null); |
| 605 | assert.isUndefined(after?.cancelledRunId); |
| 606 | return undefined; |
| 607 | } |
| 608 | // Settling the task ends the async question it left, after its own row. |
| 609 | const settled = yield* watch( |
| 610 | (event) => |
| 611 | event.type === "runtime-request.updated" && |
| 612 | event.payload.id === "request:owned" && |
| 613 | event.payload.status === "resolved", |
| 614 | ); |
| 615 | const cancelled = yield* service.cancelTask(scope, { |
| 616 | taskId: task.id, |
| 617 | clientRequestId: "cancel-task", |
| 618 | }); |
| 619 | assert.equal(cancelled.status, "cancel_requested"); |
| 620 | assert.equal((yield* parentTask)?.cancelledRunId, child.turn.runId); |
| 621 | // The turn completes normally after Stop, before its interrupt reaches the provider. |
| 622 | if (cancelAt === "turn") yield* endTurn; |
| 623 | yield* worker.drain(); |
| 624 | const owned = Effect.map(orchestrator.getThreadProjection(task.childThreadId!), (child) => |
| 625 | child.runtimeRequests.find((request) => request.id === "request:owned"), |
| 626 | ); |
| 627 | // task_cancel settles a finished turn itself; a running one settles when it ends. |
| 628 | if ((yield* owned)?.status !== "resolved") yield* Fiber.join(settled); |
| 629 | |
| 630 | const childAfter = yield* orchestrator.getThreadProjection(task.childThreadId!); |
| 631 | const nativeStatus = background === "subagent" ? "interrupted" : undefined; |
| 632 | assert.equal( |
| 633 | childAfter.subagents.find((entry) => entry.id === subagentId)?.status, |
| 634 | nativeStatus, |
| 635 | ); |
| 636 | assert.equal( |
| 637 | childAfter.nodes.find((entry) => entry.id === subagentId)?.status, |
| 638 | nativeStatus, |
| 639 | ); |
| 640 | assert.equal( |
| 641 | childAfter.turnItems.find( |
| 642 | (item) => item.type === "subagent" && item.subagentId === subagentId, |
| 643 | )?.status, |
| 644 | nativeStatus, |
| 645 | ); |
| 646 | if (remote) { |
| 647 | const descendant = childAfter.subagents.find((entry) => entry.remoteChild !== undefined); |
| 648 | assert.equal(descendant?.status, "interrupted"); |
| 649 | assert.notEqual(descendant?.remoteChild?.stoppedThere, true); |
| 650 | } |
| 651 | // Stop reached the provider turn that owns the shell. |
| 652 | if (background === "command") assert.deepEqual(interrupted, [["bg-sleep"]]); |
| 653 | assert.equal(childAfter.runs.length, cancelAt === "monitor" || cancelAt === "turn" ? 2 : 1); |
| 654 | assert.equal(childAfter.runs[0]!.status, "completed"); |
| 655 | assert.deepEqual( |
| 656 | childAfter.runtimeRequests.map((request) => [request.id, request.status]), |
| 657 | [ |
| 658 | ["request:owned", "resolved"], |
| 659 | ["request:unrelated", "pending"], |
| 660 | ], |
| 661 | ); |
| 662 | |
| 663 | const after = yield* service.taskStatus(scope, task.id); |
| 664 | assert.equal(after.status, "interrupted"); |
| 665 | assert.equal(after.workState, "result_available"); |
| 666 | const settledTask = yield* parentTask; |
| 667 | assert.equal(settledTask?.status, "interrupted"); |
| 668 | assert.notEqual(settledTask?.completedAt, null); |
| 669 | assert.equal(settledTask?.cancelledRunId, child.turn.runId); |
| 670 | |
| 671 | return { scope, taskId: task.id, settled: settledTask }; |
| 672 | }).pipe(Effect.provide(layerTest)); |
| 673 | if (stopped === undefined) return; |
| 674 | restarted = true; |
| 675 | |
| 676 | // A server restarted on the same database runs startup recovery and keeps the outcome. |
| 677 | yield* Effect.gen(function* () { |
| 678 | const service = yield* OrchestratorMcpService.OrchestratorMcpService; |
| 679 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 680 | yield* (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); |
| 681 | yield* orchestrator.recoverDelegatedTasks; |
| 682 | const task = (yield* orchestrator.getThreadProjection( |
| 683 | stopped.scope.thread!.threadId, |
| 684 | )).subagents.find((entry) => entry.id === stopped.taskId); |
| 685 | const status = yield* service.taskStatus(stopped.scope, stopped.taskId); |
| 686 | assert.equal(status.status, "interrupted"); |
| 687 | assert.equal(status.summary, task?.result); |
| 688 | assert.equal(task?.status, "interrupted"); |
| 689 | assert.notEqual(task?.completedAt, null); |
| 690 | if (stopped.settled !== undefined) assert.deepEqual(task, stopped.settled); |
| 691 | }).pipe(Effect.provide(layerTest)); |
| 692 | }), |
| 693 | ).pipe(Effect.provide(NodeServices.layer)); |
| 694 | |
| 695 | it.effect("task_cancel ends a delegated child's background native subagent", () => |
| 696 | cancelBackgroundOnlyTask("subagent"), |
| 697 | ); |
| 698 | |
| 699 | it.effect("task_cancel ends a delegated child's background shell command", () => |
| 700 | cancelBackgroundOnlyTask("command"), |
| 701 | ); |
| 702 | |
| 703 | it.effect("task_cancel ends background work while the child's run captures its checkpoint", () => |
| 704 | cancelBackgroundOnlyTask("command", { cancelAt: "checkpoint" }), |
| 705 | ); |
| 706 | |
| 707 | it.effect("task_cancel ends background work left by a turn that finishes after Stop", () => |
| 708 | cancelBackgroundOnlyTask("command", { cancelAt: "turn" }), |
| 709 | ); |
| 710 | |
| 711 | it.effect("task_cancel ends the shell's question though a monitor ran after the turn", () => |
| 712 | cancelBackgroundOnlyTask("command", { cancelAt: "monitor" }), |
| 713 | ); |
| 714 | |
| 715 | it.effect("cancellation and Stop commit together and recover before the outbox ran", () => |
| 716 | cancelBackgroundOnlyTask("subagent", { cancelAt: "recorded" }), |
| 717 | ); |
| 718 | |
| 719 | it.effect("a refused descendant stop does not settle or dispose its parent task", () => |
| 720 | cancelBackgroundOnlyTask("command", { cancelAt: "refused" }), |
| 721 | ); |
| 722 | |
| 723 | it.effect("task_cancel stops a remote descendant without reacquiring its parent lock", () => |
| 724 | cancelBackgroundOnlyTask("command", { remote: true }), |
| 725 | ); |
| 726 | |
| 727 | it.effect("a native background subagent delegates after its parent's run completes", () => |
| 728 | cancelBackgroundOnlyTask("subagent", { cancelAt: "delegate" }), |
| 729 | ); |