The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | import * as Exit from "effect/Exit"; |
| 2 | import * as EffectOutbox from "./EffectOutbox.ts"; |
| 3 | import * as EffectWorker from "./EffectWorker.ts"; |
| 4 | import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts"; |
| 5 | import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; |
| 6 | import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; |
| 7 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 8 | import { assert, it, vi } from "@effect/vitest"; |
| 9 | import { |
| 10 | CheckpointId, |
| 11 | CommandId, |
| 12 | EventId, |
| 13 | MessageId, |
| 14 | type ModelSelection, |
| 15 | NodeId, |
| 16 | type OrchestrationV2Run, |
| 17 | ProjectId, |
| 18 | ProviderDriverKind, |
| 19 | ProviderInstanceId, |
| 20 | ProviderSessionId, |
| 21 | ProviderThreadId, |
| 22 | RunId, |
| 23 | RuntimeRequestId, |
| 24 | ThreadId, |
| 25 | TurnItemId, |
| 26 | } from "@t3tools/contracts"; |
| 27 | import * as DateTime from "effect/DateTime"; |
| 28 | import * as Effect from "effect/Effect"; |
| 29 | import * as Deferred from "effect/Deferred"; |
| 30 | import * as Fiber from "effect/Fiber"; |
| 31 | import * as Layer from "effect/Layer"; |
| 32 | import * as Stream from "effect/Stream"; |
| 33 | import * as Tracer from "effect/Tracer"; |
| 34 | import * as TestClock from "effect/testing/TestClock"; |
| 35 | |
| 36 | import * as CheckpointStore from "../checkpointing/CheckpointStore.ts"; |
| 37 | import * as ServerConfig from "../config.ts"; |
| 38 | import * as McpSessionRegistryTestkit from "../mcp/McpSessionRegistry.testkit.ts"; |
| 39 | import * as SqlitePersistence from "../persistence/Sqlite.ts"; |
| 40 | import * as ProjectEnrichmentService from "../project/ProjectEnrichmentService.ts"; |
| 41 | import * as ProjectService from "../project/ProjectService.ts"; |
| 42 | import type { ProviderInstance } from "@t3tools/provider-core/server/driver"; |
| 43 | import * as ModelManifest from "../provider/ModelManifest.ts"; |
| 44 | import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts"; |
| 45 | import * as ServerSettings from "../serverSettings.ts"; |
| 46 | import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts"; |
| 47 | import * as VcsProcess from "../vcs/VcsProcess.ts"; |
| 48 | import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; |
| 49 | import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; |
| 50 | import * as EventSink from "./EventSink.ts"; |
| 51 | import * as Orchestrator from "./Orchestrator.ts"; |
| 52 | import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts"; |
| 53 | import { continueRestartedRun } from "./RestartContinuation.ts"; |
| 54 | import * as RuntimeLayer from "./runtimeLayer.ts"; |
| 55 | import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts"; |
| 56 | import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts"; |
| 57 | import * as ProviderTurnStartServiceTestkit from "./ProviderTurnStartService.testkit.ts"; |
| 58 | import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; |
| 59 | import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; |
| 60 | import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; |
| 61 | import { PROVIDER_STOP_REASON_SIGNAL } from "@t3tools/provider-core/server/subagentProjection"; |
| 62 | import * as ProviderContinuationService from "./ProviderContinuationService.ts"; |
| 63 | |
| 64 | const layerPlatformTest = Layer.merge( |
| 65 | Layer.mergeAll(NodeServices.layer, NodeCrypto.layer), |
| 66 | Layer.mock(SourceControlProviderRegistry.SourceControlProviderRegistry)({ |
| 67 | resolveLink: () => Effect.die("unused title link"), |
| 68 | }), |
| 69 | ); |
| 70 | |
| 71 | const layerServerConfig = ServerConfig.layerTest(process.cwd(), { |
| 72 | prefix: "t3-orchestration-v2-delegated-completion-", |
| 73 | }); |
| 74 | |
| 75 | const modelSelection = { |
| 76 | instanceId: ProviderInstanceId.make("codex"), |
| 77 | model: "gpt-5.4", |
| 78 | } satisfies ModelSelection; |
| 79 | |
| 80 | const layerVcsDriverRegistryTest = VcsDriverRegistry.layer.pipe( |
| 81 | Layer.provide(VcsProcess.layer), |
| 82 | Layer.provide(layerServerConfig), |
| 83 | Layer.provide(layerPlatformTest), |
| 84 | ); |
| 85 | |
| 86 | const layerCheckpointStoreTest = CheckpointStore.layer.pipe( |
| 87 | Layer.provide(layerVcsDriverRegistryTest), |
| 88 | ); |
| 89 | |
| 90 | const driver = ProviderDriverKind.make("codex"); |
| 91 | const orchestrationAdapter = { |
| 92 | instanceId: modelSelection.instanceId, |
| 93 | driver, |
| 94 | getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2), |
| 95 | planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), |
| 96 | openSession: () => Effect.die("sessions are not used by delegated completion tests"), |
| 97 | } as ProviderAdapter.ProviderAdapterV2["Service"]; |
| 98 | const providerInstance = { |
| 99 | instanceId: modelSelection.instanceId, |
| 100 | driverKind: driver, |
| 101 | continuationIdentity: { |
| 102 | driverKind: driver, |
| 103 | continuationKey: "codex:test", |
| 104 | }, |
| 105 | displayName: "Codex test", |
| 106 | enabled: true, |
| 107 | snapshot: { getSnapshot: Effect.succeed({}) } as unknown as ProviderInstance["snapshot"], |
| 108 | orchestrationAdapter, |
| 109 | textGeneration: {} as ProviderInstance["textGeneration"], |
| 110 | } satisfies ProviderInstance; |
| 111 | |
| 112 | const layerTestProviderInstanceRegistry = Layer.merge( |
| 113 | ModelManifest.layerTest, |
| 114 | Layer.succeed(ProviderInstanceRegistry.ProviderInstanceRegistry, { |
| 115 | getInstance: (instanceId) => |
| 116 | Effect.succeed(instanceId === providerInstance.instanceId ? providerInstance : undefined), |
| 117 | listInstances: Effect.succeed([providerInstance]), |
| 118 | listUnavailable: Effect.succeed([]), |
| 119 | streamChanges: Stream.empty, |
| 120 | subscribeChanges: Effect.never, |
| 121 | }), |
| 122 | ); |
| 123 | |
| 124 | const layerTest = Layer.mergeAll( |
| 125 | RuntimeLayer.layer, |
| 126 | RuntimeLayer.layerEventSink, |
| 127 | ThreadCommandExecutor.layer, |
| 128 | EffectOutbox.layer, |
| 129 | ).pipe( |
| 130 | Layer.provideMerge(RuntimeLayer.layerProjectService), |
| 131 | Layer.provide( |
| 132 | Layer.mock(WorkspacePaths.WorkspacePaths)({ |
| 133 | normalizeWorkspaceRoot: (workspaceRoot) => Effect.succeed(workspaceRoot), |
| 134 | }), |
| 135 | ), |
| 136 | Layer.provide(ProviderTurnStartServiceTestkit.layer), |
| 137 | Layer.provide( |
| 138 | Layer.succeed(ProjectEnrichmentService.ProjectEnrichmentService, { |
| 139 | peek: () => |
| 140 | Effect.succeed({ |
| 141 | repositoryIdentity: null, |
| 142 | faviconPath: null, |
| 143 | repositoryIdentityResolved: false, |
| 144 | }), |
| 145 | request: () => Effect.void, |
| 146 | readRepositoryIdentity: () => Effect.succeed(null), |
| 147 | getAvailable: () => |
| 148 | Effect.succeed({ |
| 149 | repositoryIdentity: null, |
| 150 | faviconPath: null, |
| 151 | repositoryIdentityResolved: false, |
| 152 | }), |
| 153 | invalidate: () => Effect.void, |
| 154 | refreshRepositoryIdentity: () => Effect.void, |
| 155 | subscribeChanges: Effect.never, |
| 156 | }), |
| 157 | ), |
| 158 | Layer.provide(McpSessionRegistryTestkit.layer), |
| 159 | Layer.provide(McpProviderSessions.layer), |
| 160 | Layer.provide(SqlitePersistence.layerMemory), |
| 161 | Layer.provide(layerCheckpointStoreTest), |
| 162 | Layer.provide(layerServerConfig), |
| 163 | Layer.provide(ServerSettings.layerTest()), |
| 164 | Layer.provide(layerTestProviderInstanceRegistry), |
| 165 | Layer.provide(layerPlatformTest), |
| 166 | ); |
| 167 | |
| 168 | // ProviderContinuationService consumes the orchestrator's continuation queue, as in production. |
| 169 | const layerContinuationWorkerTest = ProviderContinuationService.layer.pipe( |
| 170 | Layer.provide(Layer.merge(ProviderContinuationRequests.layer, IdAllocator.layer)), |
| 171 | Layer.provideMerge(layerTest), |
| 172 | ); |
| 173 | |
| 174 | const seedParentWithTerminalTask = (input: { |
| 175 | readonly threadId: ThreadId; |
| 176 | readonly projectId: ProjectId; |
| 177 | readonly runId: RunId; |
| 178 | readonly rootNodeId: NodeId; |
| 179 | readonly taskId: NodeId; |
| 180 | readonly deliveryState: "pending" | "delivered" | "claimed" | "acknowledged" | "disposed"; |
| 181 | readonly completionWake?: "always" | "settled_only"; |
| 182 | readonly deliveryTaskIds?: ReadonlyArray<NodeId>; |
| 183 | readonly parentRunStatus?: "running" | "completed"; |
| 184 | readonly siblingTasks?: ReadonlyArray<{ |
| 185 | readonly id: NodeId; |
| 186 | readonly deliveryState: "delivered" | "claimed"; |
| 187 | }>; |
| 188 | readonly now: DateTime.Utc; |
| 189 | }) => |
| 190 | Effect.gen(function* () { |
| 191 | const projects = yield* ProjectService.ProjectService; |
| 192 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 193 | const eventSink = yield* EventSink.EventSinkV2; |
| 194 | const providerThreadId = ProviderThreadId.make( |
| 195 | `provider-thread:${String(input.threadId).replace("thread:", "")}`, |
| 196 | ); |
| 197 | |
| 198 | yield* projects.create({ |
| 199 | commandId: CommandId.make(`command:seed-project:${input.threadId}`), |
| 200 | projectId: input.projectId, |
| 201 | title: "Delegated completion delivery", |
| 202 | workspaceRoot: `/workspace/${input.projectId}`, |
| 203 | }); |
| 204 | |
| 205 | yield* orchestrator.dispatch({ |
| 206 | type: "thread.create", |
| 207 | createdBy: "user", |
| 208 | creationSource: "web", |
| 209 | commandId: CommandId.make(`command:seed-create:${input.threadId}`), |
| 210 | threadId: input.threadId, |
| 211 | projectId: input.projectId, |
| 212 | title: "Delegated completion delivery", |
| 213 | modelSelection, |
| 214 | runtimeMode: "full-access", |
| 215 | interactionMode: "default", |
| 216 | branch: null, |
| 217 | worktreePath: null, |
| 218 | }); |
| 219 | |
| 220 | yield* eventSink.write({ |
| 221 | commandId: CommandId.make(`command:seed-projection:${input.threadId}`), |
| 222 | events: [ |
| 223 | { |
| 224 | id: EventId.make(`event:seed-provider-thread:${input.threadId}`), |
| 225 | type: "provider-thread.updated", |
| 226 | threadId: input.threadId, |
| 227 | driver, |
| 228 | providerInstanceId: modelSelection.instanceId, |
| 229 | occurredAt: input.now, |
| 230 | payload: { |
| 231 | id: providerThreadId, |
| 232 | driver, |
| 233 | providerInstanceId: modelSelection.instanceId, |
| 234 | providerSessionId: null, |
| 235 | appThreadId: input.threadId, |
| 236 | ownerNodeId: input.rootNodeId, |
| 237 | nativeThreadRef: { |
| 238 | driver, |
| 239 | nativeId: `native:${input.threadId}`, |
| 240 | strength: "strong", |
| 241 | }, |
| 242 | nativeConversationHeadRef: null, |
| 243 | status: "active", |
| 244 | firstRunOrdinal: 1, |
| 245 | lastRunOrdinal: 1, |
| 246 | handoffIds: [], |
| 247 | forkedFrom: null, |
| 248 | createdAt: input.now, |
| 249 | updatedAt: input.now, |
| 250 | }, |
| 251 | }, |
| 252 | { |
| 253 | id: EventId.make(`event:seed-run:${input.threadId}`), |
| 254 | type: "run.updated", |
| 255 | threadId: input.threadId, |
| 256 | runId: input.runId, |
| 257 | nodeId: input.rootNodeId, |
| 258 | providerInstanceId: modelSelection.instanceId, |
| 259 | occurredAt: input.now, |
| 260 | payload: { |
| 261 | id: input.runId, |
| 262 | threadId: input.threadId, |
| 263 | ordinal: 1, |
| 264 | providerInstanceId: modelSelection.instanceId, |
| 265 | modelSelection, |
| 266 | providerThreadId, |
| 267 | userMessageId: MessageId.make(`message:seed-user:${input.threadId}`), |
| 268 | rootNodeId: input.rootNodeId, |
| 269 | activeAttemptId: null, |
| 270 | status: input.parentRunStatus ?? "running", |
| 271 | requestedAt: input.now, |
| 272 | startedAt: input.now, |
| 273 | completedAt: input.parentRunStatus === "completed" ? input.now : null, |
| 274 | checkpointId: null, |
| 275 | contextHandoffId: null, |
| 276 | delegatedCompletion: { |
| 277 | disposition: "open", |
| 278 | nextGeneration: 2, |
| 279 | delivery: |
| 280 | input.deliveryTaskIds === undefined |
| 281 | ? null |
| 282 | : { |
| 283 | generation: 1, |
| 284 | messageId: MessageId.make(`message:delegated-delivery:${input.threadId}`), |
| 285 | taskIds: input.deliveryTaskIds, |
| 286 | }, |
| 287 | }, |
| 288 | }, |
| 289 | }, |
| 290 | ...[ |
| 291 | { id: input.taskId, deliveryState: input.deliveryState }, |
| 292 | ...(input.siblingTasks ?? []), |
| 293 | ].map((task) => ({ |
| 294 | id: EventId.make(`event:seed-task:${input.threadId}:${task.id}`), |
| 295 | type: "subagent.updated" as const, |
| 296 | threadId: input.threadId, |
| 297 | runId: input.runId, |
| 298 | nodeId: task.id, |
| 299 | driver, |
| 300 | providerInstanceId: modelSelection.instanceId, |
| 301 | occurredAt: input.now, |
| 302 | payload: { |
| 303 | id: task.id, |
| 304 | threadId: input.threadId, |
| 305 | runId: input.runId, |
| 306 | parentNodeId: input.rootNodeId, |
| 307 | origin: "app_owned" as const, |
| 308 | createdBy: "agent" as const, |
| 309 | driver, |
| 310 | providerInstanceId: modelSelection.instanceId, |
| 311 | providerThreadId: null, |
| 312 | childThreadId: null, |
| 313 | nativeTaskRef: null, |
| 314 | prompt: "Inspect the delivered ownership edge.", |
| 315 | title: null, |
| 316 | model: null, |
| 317 | completionWake: input.completionWake ?? "settled_only", |
| 318 | completionDelivery: { |
| 319 | state: task.deliveryState, |
| 320 | observedByRunId: task.deliveryState === "acknowledged" ? input.runId : null, |
| 321 | }, |
| 322 | status: "completed" as const, |
| 323 | result: "child finished", |
| 324 | startedAt: input.now, |
| 325 | completedAt: input.now, |
| 326 | updatedAt: input.now, |
| 327 | }, |
| 328 | })), |
| 329 | ], |
| 330 | }); |
| 331 | }); |
| 332 | |
| 333 | it.layer(layerTest)("delegated completion delivery repairs", (it) => { |
| 334 | it.effect( |
| 335 | "interrupting a completion wake disposes both cohorts without reviving them on late results", |
| 336 | () => |
| 337 | Effect.gen(function* () { |
| 338 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 339 | const sink = yield* EventSink.EventSinkV2; |
| 340 | const now = yield* DateTime.now; |
| 341 | const threadId = ThreadId.make("stop-wake-parent"); |
| 342 | const originRunId = RunId.make("stop-wake-origin"); |
| 343 | const incomingTaskId = NodeId.make("stop-wake-incoming"); |
| 344 | yield* seedParentWithTerminalTask({ |
| 345 | threadId, |
| 346 | projectId: ProjectId.make("stop-wake-project"), |
| 347 | runId: originRunId, |
| 348 | rootNodeId: NodeId.make("stop-wake-origin-root"), |
| 349 | taskId: incomingTaskId, |
| 350 | deliveryState: "claimed", |
| 351 | deliveryTaskIds: [incomingTaskId], |
| 352 | completionWake: "always", |
| 353 | now, |
| 354 | }); |
| 355 | const origin = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!; |
| 356 | yield* sink.write({ |
| 357 | events: [ |
| 358 | { |
| 359 | ...runEvent({ threadId, runId: originRunId, ordinal: 1, status: "completed", now }), |
| 360 | payload: { ...origin, status: "completed", completedAt: now }, |
| 361 | }, |
| 362 | ], |
| 363 | }); |
| 364 | const delivery = origin.delegatedCompletion!.delivery!; |
| 365 | yield* orchestrator.dispatch({ |
| 366 | type: "message.dispatch", |
| 367 | createdBy: "agent", |
| 368 | creationSource: "server", |
| 369 | commandId: CommandId.make("stop-wake-start"), |
| 370 | threadId, |
| 371 | messageId: delivery.messageId, |
| 372 | text: "Delegated task completed", |
| 373 | attachments: [], |
| 374 | modelSelection, |
| 375 | dispatchMode: { type: "queue_after_active" }, |
| 376 | delegatedCompletion: { |
| 377 | parentRunId: originRunId, |
| 378 | generation: delivery.generation, |
| 379 | taskIds: delivery.taskIds, |
| 380 | }, |
| 381 | }); |
| 382 | yield* orchestrator.resumeQueuedRuns; |
| 383 | const wake = (yield* orchestrator.getThreadProjection(threadId)).runs.find( |
| 384 | (run) => run.userMessageId === delivery.messageId, |
| 385 | )!; |
| 386 | yield* orchestrator.dispatch({ |
| 387 | type: "delegated_task.request", |
| 388 | createdBy: "agent", |
| 389 | creationSource: "mcp", |
| 390 | commandId: CommandId.make("stop-wake-delegate"), |
| 391 | parentThreadId: threadId, |
| 392 | parentRunId: wake.id, |
| 393 | parentNodeId: wake.rootNodeId!, |
| 394 | task: "Do the next round", |
| 395 | modelSelection, |
| 396 | runtimeMode: "full-access", |
| 397 | interactionMode: "default", |
| 398 | completionWake: "always", |
| 399 | }); |
| 400 | const delegated = (yield* orchestrator.getThreadProjection(threadId)).subagents.find( |
| 401 | (task) => task.runId === wake.id, |
| 402 | )!; |
| 403 | yield* orchestrator.dispatch({ |
| 404 | type: "run.interrupt", |
| 405 | commandId: CommandId.make("stop-wake-interrupt"), |
| 406 | threadId, |
| 407 | runId: wake.id, |
| 408 | }); |
| 409 | const stopped = yield* orchestrator.getThreadProjection(threadId); |
| 410 | for (const runId of [originRunId, wake.id]) { |
| 411 | const run = stopped.runs.find((row) => row.id === runId)!; |
| 412 | assert.equal(run.delegatedCompletion?.disposition, "stopped"); |
| 413 | assert.isNull(run.delegatedCompletion?.delivery); |
| 414 | } |
| 415 | assert.equal(stopped.runs.find((run) => run.id === wake.id)?.status, "interrupted"); |
| 416 | for (const taskId of [incomingTaskId, delegated.id]) { |
| 417 | assert.equal( |
| 418 | stopped.subagents.find((task) => task.id === taskId)?.completionDelivery?.state, |
| 419 | "disposed", |
| 420 | ); |
| 421 | } |
| 422 | assert.equal( |
| 423 | stopped.subagents.find((task) => task.id === incomingTaskId)?.result, |
| 424 | "child finished", |
| 425 | ); |
| 426 | |
| 427 | yield* orchestrator.dispatch({ |
| 428 | type: "message.dispatch", |
| 429 | createdBy: "user", |
| 430 | creationSource: "web", |
| 431 | commandId: CommandId.make("stop-wake-newer-work"), |
| 432 | threadId, |
| 433 | messageId: MessageId.make("stop-wake-newer-work"), |
| 434 | text: "Start unrelated work", |
| 435 | attachments: [], |
| 436 | modelSelection, |
| 437 | dispatchMode: { type: "queue_after_active" }, |
| 438 | }); |
| 439 | yield* orchestrator.resumeQueuedRuns; |
| 440 | const newer = yield* orchestrator.getThreadProjection(threadId); |
| 441 | assert.equal(newer.runs.at(-1)?.status, "starting"); |
| 442 | |
| 443 | const childThreadId = delegated.childThreadId!; |
| 444 | const childRun = (yield* orchestrator.getThreadProjection(childThreadId)).runs[0]!; |
| 445 | const afterSequence = yield* sink.latestSequence(); |
| 446 | yield* sink.write({ |
| 447 | events: [ |
| 448 | { |
| 449 | ...runEvent({ |
| 450 | threadId: childThreadId, |
| 451 | runId: childRun.id, |
| 452 | ordinal: childRun.ordinal, |
| 453 | status: "completed", |
| 454 | now, |
| 455 | }), |
| 456 | payload: { ...childRun, status: "completed", completedAt: now }, |
| 457 | }, |
| 458 | ], |
| 459 | }); |
| 460 | yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe( |
| 461 | Stream.filter( |
| 462 | (stored) => |
| 463 | stored.event.type === "subagent.updated" && |
| 464 | stored.event.payload.id === delegated.id && |
| 465 | stored.event.payload.status === "completed", |
| 466 | ), |
| 467 | Stream.take(1), |
| 468 | Stream.runDrain, |
| 469 | ); |
| 470 | const finished = yield* orchestrator.getThreadProjection(threadId); |
| 471 | const lateTask = finished.subagents.find((task) => task.id === delegated.id)!; |
| 472 | assert.equal(lateTask.status, "completed"); |
| 473 | assert.isNotNull(lateTask.result); |
| 474 | assert.equal(lateTask.completionDelivery?.state, "disposed"); |
| 475 | assert.deepEqual(finished.runs, newer.runs); |
| 476 | assert.deepEqual(finished.messages, newer.messages); |
| 477 | }), |
| 478 | ); |
| 479 | |
| 480 | it.effect("acceptance batches pending siblings without acknowledging their results", () => |
| 481 | Effect.gen(function* () { |
| 482 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 483 | const sink = yield* EventSink.EventSinkV2; |
| 484 | const now = yield* DateTime.now; |
| 485 | const threadId = ThreadId.make("mailbox-batch"); |
| 486 | const runId = RunId.make("mailbox-parent"); |
| 487 | const taskId = NodeId.make("mailbox-first"); |
| 488 | const messageId = MessageId.make(`message:delegated-delivery:${threadId}`); |
| 489 | yield* seedParentWithTerminalTask({ |
| 490 | threadId, |
| 491 | runId, |
| 492 | projectId: ProjectId.make("mailbox-project"), |
| 493 | rootNodeId: NodeId.make("mailbox-root"), |
| 494 | taskId, |
| 495 | deliveryState: "claimed", |
| 496 | completionWake: "always", |
| 497 | deliveryTaskIds: [taskId], |
| 498 | now, |
| 499 | }); |
| 500 | const projection = yield* orchestrator.getThreadProjection(threadId); |
| 501 | const task = projection.subagents[0]!; |
| 502 | const pendingIds = [NodeId.make("mailbox-second"), NodeId.make("mailbox-third")]; |
| 503 | yield* sink.write({ |
| 504 | events: [ |
| 505 | { |
| 506 | id: EventId.make("mailbox-message"), |
| 507 | type: "message.updated", |
| 508 | threadId, |
| 509 | runId, |
| 510 | occurredAt: now, |
| 511 | payload: { |
| 512 | id: messageId, |
| 513 | threadId, |
| 514 | runId, |
| 515 | nodeId: task.parentNodeId, |
| 516 | role: "user", |
| 517 | text: "Background task finished", |
| 518 | attachments: [], |
| 519 | streaming: false, |
| 520 | createdBy: "agent", |
| 521 | creationSource: "server", |
| 522 | createdAt: now, |
| 523 | updatedAt: now, |
| 524 | delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [taskId] }, |
| 525 | }, |
| 526 | }, |
| 527 | ...pendingIds.map((id) => ({ |
| 528 | id: EventId.make(`event:${id}`), |
| 529 | type: "subagent.updated" as const, |
| 530 | threadId, |
| 531 | runId, |
| 532 | nodeId: id, |
| 533 | occurredAt: now, |
| 534 | payload: { |
| 535 | ...task, |
| 536 | id, |
| 537 | completionDelivery: { state: "pending" as const, observedByRunId: null }, |
| 538 | }, |
| 539 | })), |
| 540 | ], |
| 541 | }); |
| 542 | yield* orchestrator.dispatch({ |
| 543 | type: "notification.delivery.accept", |
| 544 | commandId: CommandId.make("accept-first"), |
| 545 | threadId, |
| 546 | messageId, |
| 547 | }); |
| 548 | const accepted = yield* orchestrator.getThreadProjection(threadId); |
| 549 | assert.equal( |
| 550 | accepted.subagents.find((row) => row.id === taskId)?.completionDelivery?.state, |
| 551 | "delivered", |
| 552 | ); |
| 553 | const cohort = accepted.runs.find((row) => row.id === runId)?.delegatedCompletion; |
| 554 | assert.deepEqual(cohort?.delivery?.taskIds, pendingIds); |
| 555 | assert.equal(cohort?.delivery?.generation, 2); |
| 556 | for (const id of pendingIds) { |
| 557 | assert.deepEqual(accepted.subagents.find((row) => row.id === id)?.completionDelivery, { |
| 558 | state: "claimed", |
| 559 | observedByRunId: null, |
| 560 | }); |
| 561 | } |
| 562 | yield* orchestrator.dispatch({ |
| 563 | type: "notification.delivery.accept", |
| 564 | commandId: CommandId.make("repeat-old-acceptance"), |
| 565 | threadId, |
| 566 | messageId, |
| 567 | }); |
| 568 | const duplicate = yield* orchestrator.getThreadProjection(threadId); |
| 569 | assert.deepEqual(duplicate.runs.find((row) => row.id === runId)?.delegatedCompletion, cohort); |
| 570 | }), |
| 571 | ); |
| 572 | |
| 573 | it.effect("acceptance batches a settled_only sibling once its spawning run ended", () => |
| 574 | Effect.gen(function* () { |
| 575 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 576 | const sink = yield* EventSink.EventSinkV2; |
| 577 | const now = yield* DateTime.now; |
| 578 | const threadId = ThreadId.make("mailbox-mixed"); |
| 579 | const runId = RunId.make("mailbox-mixed-parent"); |
| 580 | const taskId = NodeId.make("mailbox-mixed-always"); |
| 581 | const siblingId = NodeId.make("mailbox-mixed-settled-only"); |
| 582 | const messageId = MessageId.make(`message:delegated-delivery:${threadId}`); |
| 583 | yield* seedParentWithTerminalTask({ |
| 584 | threadId, |
| 585 | runId, |
| 586 | projectId: ProjectId.make("mailbox-mixed-project"), |
| 587 | rootNodeId: NodeId.make("mailbox-mixed-root"), |
| 588 | taskId, |
| 589 | deliveryState: "claimed", |
| 590 | completionWake: "always", |
| 591 | deliveryTaskIds: [taskId], |
| 592 | now, |
| 593 | }); |
| 594 | const projection = yield* orchestrator.getThreadProjection(threadId); |
| 595 | const task = projection.subagents[0]!; |
| 596 | const spawningRun = projection.runs.find((row) => row.id === runId)!; |
| 597 | yield* sink.write({ |
| 598 | events: [ |
| 599 | { |
| 600 | ...runEvent({ threadId, runId, ordinal: 1, status: "cancelled", now }), |
| 601 | payload: { ...spawningRun, status: "cancelled" as const, completedAt: now }, |
| 602 | }, |
| 603 | runEvent({ |
| 604 | threadId, |
| 605 | runId: RunId.make("mailbox-mixed-continuation"), |
| 606 | ordinal: 2, |
| 607 | status: "running", |
| 608 | now, |
| 609 | }), |
| 610 | { |
| 611 | id: EventId.make("mailbox-mixed-message"), |
| 612 | type: "message.updated", |
| 613 | threadId, |
| 614 | runId, |
| 615 | occurredAt: now, |
| 616 | payload: { |
| 617 | id: messageId, |
| 618 | threadId, |
| 619 | runId, |
| 620 | nodeId: task.parentNodeId, |
| 621 | role: "user", |
| 622 | text: "Background task finished", |
| 623 | attachments: [], |
| 624 | streaming: false, |
| 625 | createdBy: "agent", |
| 626 | creationSource: "server", |
| 627 | createdAt: now, |
| 628 | updatedAt: now, |
| 629 | delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [taskId] }, |
| 630 | }, |
| 631 | }, |
| 632 | { |
| 633 | id: EventId.make(`event:${siblingId}`), |
| 634 | type: "subagent.updated" as const, |
| 635 | threadId, |
| 636 | runId, |
| 637 | nodeId: siblingId, |
| 638 | occurredAt: now, |
| 639 | payload: { |
| 640 | ...task, |
| 641 | id: siblingId, |
| 642 | completionWake: "settled_only" as const, |
| 643 | completionDelivery: { state: "pending" as const, observedByRunId: null }, |
| 644 | }, |
| 645 | }, |
| 646 | ], |
| 647 | }); |
| 648 | yield* orchestrator.dispatch({ |
| 649 | type: "notification.delivery.accept", |
| 650 | commandId: CommandId.make("accept-mixed"), |
| 651 | threadId, |
| 652 | messageId, |
| 653 | }); |
| 654 | const accepted = yield* orchestrator.getThreadProjection(threadId); |
| 655 | const cohort = accepted.runs.find((row) => row.id === runId)?.delegatedCompletion; |
| 656 | assert.deepEqual(cohort?.delivery?.taskIds, [siblingId]); |
| 657 | assert.equal( |
| 658 | accepted.subagents.find((row) => row.id === siblingId)?.completionDelivery?.state, |
| 659 | "claimed", |
| 660 | ); |
| 661 | }), |
| 662 | ); |
| 663 | |
| 664 | it.effect("builds completion text and metadata from the same live cohort", () => |
| 665 | Effect.gen(function* () { |
| 666 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 667 | const now = yield* DateTime.now; |
| 668 | const threadId = ThreadId.make("thread:delegated-delivery-live-cohort"); |
| 669 | const projectId = ProjectId.make("project:delegated-delivery-live-cohort"); |
| 670 | const runId = RunId.make("run:delegated-delivery-live-cohort"); |
| 671 | const rootNodeId = NodeId.make("node:delegated-delivery-live-cohort-root"); |
| 672 | const firstTaskId = NodeId.make("node:delegated-delivery-live-cohort-first"); |
| 673 | const secondTaskId = NodeId.make("node:delegated-delivery-live-cohort-second"); |
| 674 | const messageId = MessageId.make(`message:delegated-delivery:${threadId}`); |
| 675 | |
| 676 | yield* seedParentWithTerminalTask({ |
| 677 | threadId, |
| 678 | projectId, |
| 679 | runId, |
| 680 | rootNodeId, |
| 681 | taskId: firstTaskId, |
| 682 | deliveryState: "claimed", |
| 683 | completionWake: "always", |
| 684 | deliveryTaskIds: [firstTaskId, secondTaskId], |
| 685 | now, |
| 686 | }); |
| 687 | |
| 688 | yield* orchestrator.dispatch({ |
| 689 | type: "message.dispatch", |
| 690 | commandId: CommandId.make("command:delegated-delivery-live-cohort"), |
| 691 | threadId, |
| 692 | messageId, |
| 693 | text: `Delegated task ${firstTaskId} reached a terminal state.`, |
| 694 | attachments: [], |
| 695 | dispatchMode: { type: "queue_after_active" }, |
| 696 | createdBy: "agent", |
| 697 | creationSource: "server", |
| 698 | delegatedCompletion: { |
| 699 | parentRunId: runId, |
| 700 | generation: 1, |
| 701 | taskIds: [firstTaskId], |
| 702 | }, |
| 703 | }); |
| 704 | |
| 705 | const projection = yield* orchestrator.getThreadProjection(threadId); |
| 706 | const message = projection.messages.find((candidate) => candidate.id === messageId); |
| 707 | assert.deepEqual(message?.delegatedCompletion?.taskIds, [firstTaskId, secondTaskId]); |
| 708 | assert.include(message?.text ?? "", String(firstTaskId)); |
| 709 | assert.include(message?.text ?? "", String(secondTaskId)); |
| 710 | assert.include(message?.text ?? "", "task_status"); |
| 711 | assert.include(message?.text ?? "", "child finished"); |
| 712 | }), |
| 713 | ); |
| 714 | |
| 715 | it.effect("counts already delivered siblings in the completion notice", () => |
| 716 | Effect.gen(function* () { |
| 717 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 718 | const now = yield* DateTime.now; |
| 719 | const threadId = ThreadId.make("thread:delegated-delivery-notice-count"); |
| 720 | const runId = RunId.make("run:delegated-delivery-notice-count"); |
| 721 | const firstTaskId = NodeId.make("node:delegated-delivery-notice-count-first"); |
| 722 | const secondTaskId = NodeId.make("node:delegated-delivery-notice-count-second"); |
| 723 | |
| 724 | yield* seedParentWithTerminalTask({ |
| 725 | threadId, |
| 726 | projectId: ProjectId.make("project:delegated-delivery-notice-count"), |
| 727 | runId, |
| 728 | rootNodeId: NodeId.make("node:delegated-delivery-notice-count-root"), |
| 729 | taskId: firstTaskId, |
| 730 | deliveryState: "claimed", |
| 731 | completionWake: "always", |
| 732 | deliveryTaskIds: [firstTaskId, secondTaskId], |
| 733 | parentRunStatus: "completed", |
| 734 | siblingTasks: [ |
| 735 | { id: secondTaskId, deliveryState: "claimed" }, |
| 736 | { |
| 737 | id: NodeId.make("node:delegated-delivery-notice-count-earlier"), |
| 738 | deliveryState: "delivered", |
| 739 | }, |
| 740 | ], |
| 741 | now, |
| 742 | }); |
| 743 | |
| 744 | yield* orchestrator.dispatch({ |
| 745 | type: "message.dispatch", |
| 746 | commandId: CommandId.make("command:delegated-delivery-notice-count"), |
| 747 | threadId, |
| 748 | messageId: MessageId.make(`message:delegated-delivery:${threadId}`), |
| 749 | text: "Delegated tasks reached a terminal state.", |
| 750 | attachments: [], |
| 751 | dispatchMode: { type: "queue_after_active" }, |
| 752 | createdBy: "agent", |
| 753 | creationSource: "server", |
| 754 | delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [firstTaskId] }, |
| 755 | }); |
| 756 | |
| 757 | const projection = yield* orchestrator.getThreadProjection(threadId); |
| 758 | const notice = projection.turnItems.find((item) => item.type === "notification"); |
| 759 | assert.match( |
| 760 | notice?.type === "notification" ? notice.summary : "", |
| 761 | /^2 of 3 delegated tasks finished:/, |
| 762 | ); |
| 763 | }), |
| 764 | ); |
| 765 | |
| 766 | it.effect.each([ |
| 767 | { name: "no result is waiting", delivery: "none", compacts: true }, |
| 768 | { name: "a result is waiting to be sent", delivery: "pending", compacts: false }, |
| 769 | { name: "a result was just sent", delivery: "sent", compacts: false }, |
| 770 | ] as const)("compacts before the cache expires only when $name", ({ delivery, compacts }) => |
| 771 | Effect.gen(function* () { |
| 772 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 773 | const now = yield* DateTime.now; |
| 774 | const threadId = ThreadId.make(`thread:idle-compaction-delivery-${delivery}`); |
| 775 | const runId = RunId.make(`run:idle-compaction-delivery-${delivery}`); |
| 776 | const taskId = NodeId.make(`node:idle-compaction-delivery-${delivery}-task`); |
| 777 | |
| 778 | yield* seedParentWithTerminalTask({ |
| 779 | threadId, |
| 780 | projectId: ProjectId.make(`project:idle-compaction-delivery-${delivery}`), |
| 781 | runId, |
| 782 | rootNodeId: NodeId.make(`node:idle-compaction-delivery-${delivery}-root`), |
| 783 | taskId, |
| 784 | deliveryState: delivery === "none" ? "delivered" : "claimed", |
| 785 | completionWake: "always", |
| 786 | parentRunStatus: "completed", |
| 787 | ...(delivery === "none" ? {} : { deliveryTaskIds: [taskId] }), |
| 788 | now, |
| 789 | }); |
| 790 | if (delivery === "sent") { |
| 791 | yield* orchestrator.dispatch({ |
| 792 | type: "message.dispatch", |
| 793 | commandId: CommandId.make(`command:idle-compaction-delivery-${delivery}`), |
| 794 | threadId, |
| 795 | messageId: MessageId.make(`message:delegated-delivery:${threadId}`), |
| 796 | text: `Delegated task ${taskId} reached a terminal state.`, |
| 797 | attachments: [], |
| 798 | dispatchMode: { type: "queue_after_active" }, |
| 799 | createdBy: "agent", |
| 800 | creationSource: "server", |
| 801 | delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [taskId] }, |
| 802 | }); |
| 803 | } |
| 804 | |
| 805 | const compactionId = MessageId.make(`idle-compact:${threadId}:${runId}`); |
| 806 | yield* orchestrator.dispatch({ |
| 807 | type: "message.dispatch", |
| 808 | commandId: CommandId.make(`server:idle-compact:${threadId}:${runId}`), |
| 809 | threadId, |
| 810 | messageId: compactionId, |
| 811 | text: "/compact", |
| 812 | attachments: [], |
| 813 | createdBy: "agent", |
| 814 | creationSource: "server", |
| 815 | idleCompaction: "beforeCacheExpires", |
| 816 | dispatchMode: { type: "start_immediately" }, |
| 817 | }); |
| 818 | |
| 819 | const projection = yield* orchestrator.getThreadProjection(threadId); |
| 820 | assert.equal( |
| 821 | projection.messages.some((message) => message.id === compactionId), |
| 822 | compacts, |
| 823 | ); |
| 824 | }), |
| 825 | ); |
| 826 | |
| 827 | it.effect.each([ |
| 828 | { |
| 829 | name: "oversized UTF-8 result", |
| 830 | first: "界".repeat(6_000), |
| 831 | second: "small result", |
| 832 | inlineFirst: false, |
| 833 | inlineSecond: true, |
| 834 | }, |
| 835 | { |
| 836 | name: "combined results", |
| 837 | first: "a".repeat(10_000), |
| 838 | second: "b".repeat(10_000), |
| 839 | inlineFirst: true, |
| 840 | inlineSecond: false, |
| 841 | }, |
| 842 | ])("uses task_status for $name beyond the inline budget", (testCase) => |
| 843 | Effect.gen(function* () { |
| 844 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 845 | const sink = yield* EventSink.EventSinkV2; |
| 846 | const now = yield* DateTime.now; |
| 847 | const threadId = ThreadId.make(`thread:inline-budget:${testCase.name}`); |
| 848 | const runId = RunId.make(`run:inline-budget:${testCase.name}`); |
| 849 | const firstTaskId = NodeId.make(`node:inline-first:${testCase.name}`); |
| 850 | const secondTaskId = NodeId.make(`node:inline-second:${testCase.name}`); |
| 851 | const messageId = MessageId.make(`message:delegated-delivery:${threadId}`); |
| 852 | yield* seedParentWithTerminalTask({ |
| 853 | threadId, |
| 854 | projectId: ProjectId.make(`project:inline-budget:${testCase.name}`), |
| 855 | runId, |
| 856 | rootNodeId: NodeId.make(`node:inline-budget:${testCase.name}`), |
| 857 | taskId: firstTaskId, |
| 858 | deliveryState: "claimed", |
| 859 | completionWake: "always", |
| 860 | deliveryTaskIds: [firstTaskId, secondTaskId], |
| 861 | now, |
| 862 | }); |
| 863 | const parent = yield* orchestrator.getThreadProjection(threadId); |
| 864 | const task = parent.subagents[0]!; |
| 865 | yield* sink.write({ |
| 866 | events: [ |
| 867 | { id: firstTaskId, result: testCase.first, status: "failed" as const }, |
| 868 | { id: secondTaskId, result: testCase.second, status: "cancelled" as const }, |
| 869 | ].map(({ id, result, status }) => ({ |
| 870 | id: EventId.make(`event:inline-budget:${id}`), |
| 871 | type: "subagent.updated" as const, |
| 872 | threadId, |
| 873 | runId, |
| 874 | nodeId: id, |
| 875 | occurredAt: now, |
| 876 | payload: { |
| 877 | ...task, |
| 878 | id, |
| 879 | result, |
| 880 | status, |
| 881 | title: "Inspect regression", |
| 882 | childThreadId: ThreadId.make(`thread:inline-budget-child:${id}`), |
| 883 | }, |
| 884 | })), |
| 885 | }); |
| 886 | yield* orchestrator.dispatch({ |
| 887 | type: "message.dispatch", |
| 888 | commandId: CommandId.make(`command:inline-budget:${testCase.name}`), |
| 889 | threadId, |
| 890 | messageId, |
| 891 | text: "Delegated tasks completed.", |
| 892 | attachments: [], |
| 893 | dispatchMode: { type: "queue_after_active" }, |
| 894 | createdBy: "agent", |
| 895 | creationSource: "server", |
| 896 | delegatedCompletion: { |
| 897 | parentRunId: runId, |
| 898 | generation: 1, |
| 899 | taskIds: [firstTaskId, secondTaskId], |
| 900 | }, |
| 901 | }); |
| 902 | const projection = yield* orchestrator.getThreadProjection(threadId); |
| 903 | const message = projection.messages.find((candidate) => candidate.id === messageId)!; |
| 904 | assert.equal(message.text.includes(testCase.first), testCase.inlineFirst); |
| 905 | assert.equal(message.text.includes(testCase.second), testCase.inlineSecond); |
| 906 | const deferredTaskId = testCase.inlineFirst ? secondTaskId : firstTaskId; |
| 907 | assert.include( |
| 908 | message.text, |
| 909 | `Delegated task ${deferredTaskId} (Inspect regression) in child thread thread:inline-budget-child:${deferredTaskId} [${testCase.inlineFirst ? "cancelled" : "failed"}]`, |
| 910 | ); |
| 911 | assert.include(message.text, `Use task_status with taskId ${deferredTaskId}`); |
| 912 | assert.deepEqual(message.delegatedCompletion?.taskIds, [firstTaskId, secondTaskId]); |
| 913 | assert.equal( |
| 914 | projection.subagents.find((candidate) => candidate.id === deferredTaskId)?.result, |
| 915 | testCase.inlineFirst ? testCase.second : testCase.first, |
| 916 | ); |
| 917 | }), |
| 918 | ); |
| 919 | |
| 920 | it.effect("does not re-offer when wake-policy upgrades after delivered ownership settled", () => |
| 921 | Effect.gen(function* () { |
| 922 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 923 | const now = yield* DateTime.now; |
| 924 | const threadId = ThreadId.make("thread:delegated-delivery-a1"); |
| 925 | const projectId = ProjectId.make("project:delegated-delivery-a1"); |
| 926 | const runId = RunId.make("run:delegated-delivery-a1"); |
| 927 | const rootNodeId = NodeId.make("node:delegated-delivery-a1-root"); |
| 928 | const taskId = NodeId.make("node:delegated-delivery-a1-task"); |
| 929 | |
| 930 | yield* seedParentWithTerminalTask({ |
| 931 | threadId, |
| 932 | projectId, |
| 933 | runId, |
| 934 | rootNodeId, |
| 935 | taskId, |
| 936 | deliveryState: "delivered", |
| 937 | completionWake: "settled_only", |
| 938 | now, |
| 939 | }); |
| 940 | |
| 941 | const upgrade = yield* orchestrator.dispatch({ |
| 942 | type: "delegated_task.wake-policy", |
| 943 | commandId: CommandId.make("command:delegated-delivery-a1:wake-policy"), |
| 944 | parentThreadId: threadId, |
| 945 | taskId, |
| 946 | completionWake: "always", |
| 947 | }); |
| 948 | |
| 949 | const projection = yield* orchestrator.getThreadProjection(threadId); |
| 950 | const task = projection.subagents.find((candidate) => candidate.id === taskId); |
| 951 | const parentRun = projection.runs.find((candidate) => candidate.id === runId); |
| 952 | |
| 953 | assert.equal(task?.completionWake, "always"); |
| 954 | assert.deepEqual(task?.completionDelivery, { |
| 955 | state: "delivered", |
| 956 | observedByRunId: null, |
| 957 | }); |
| 958 | assert.deepEqual(parentRun?.delegatedCompletion, { |
| 959 | disposition: "open", |
| 960 | nextGeneration: 2, |
| 961 | delivery: null, |
| 962 | }); |
| 963 | assert.isFalse( |
| 964 | upgrade.storedEvents.some( |
| 965 | (stored) => |
| 966 | stored.event.type === "subagent.updated" && |
| 967 | stored.event.payload.id === taskId && |
| 968 | stored.event.payload.completionDelivery?.state === "claimed", |
| 969 | ), |
| 970 | ); |
| 971 | assert.isFalse( |
| 972 | upgrade.storedEvents.some( |
| 973 | (stored) => |
| 974 | stored.event.type === "run.updated" && |
| 975 | stored.event.payload.id === runId && |
| 976 | stored.event.payload.delegatedCompletion?.delivery !== null && |
| 977 | stored.event.payload.delegatedCompletion?.delivery !== undefined, |
| 978 | ), |
| 979 | ); |
| 980 | }), |
| 981 | ); |
| 982 | |
| 983 | it.effect( |
| 984 | "treats repeated acknowledge and dispose with distinct command IDs as successful no-ops", |
| 985 | () => |
| 986 | Effect.gen(function* () { |
| 987 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 988 | const now = yield* DateTime.now; |
| 989 | const threadId = ThreadId.make("thread:delegated-delivery-a2"); |
| 990 | const projectId = ProjectId.make("project:delegated-delivery-a2"); |
| 991 | const runId = RunId.make("run:delegated-delivery-a2"); |
| 992 | const rootNodeId = NodeId.make("node:delegated-delivery-a2-root"); |
| 993 | const taskId = NodeId.make("node:delegated-delivery-a2-task"); |
| 994 | |
| 995 | yield* seedParentWithTerminalTask({ |
| 996 | threadId, |
| 997 | projectId, |
| 998 | runId, |
| 999 | rootNodeId, |
| 1000 | taskId, |
| 1001 | deliveryState: "delivered", |
| 1002 | completionWake: "always", |
| 1003 | now, |
| 1004 | }); |
| 1005 | |
| 1006 | const firstAck = yield* orchestrator.dispatch({ |
| 1007 | type: "delegated_task.completion-delivery.acknowledge", |
| 1008 | commandId: CommandId.make("command:delegated-delivery-a2:ack-task-status"), |
| 1009 | parentThreadId: threadId, |
| 1010 | taskId, |
| 1011 | observedByRunId: runId, |
| 1012 | }); |
| 1013 | const secondAck = yield* orchestrator.dispatch({ |
| 1014 | type: "delegated_task.completion-delivery.acknowledge", |
| 1015 | commandId: CommandId.make("command:delegated-delivery-a2:ack-thread-read"), |
| 1016 | parentThreadId: threadId, |
| 1017 | taskId, |
| 1018 | observedByRunId: runId, |
| 1019 | }); |
| 1020 | |
| 1021 | const firstAckTask = firstAck.storedEvents.find( |
| 1022 | (stored) => |
| 1023 | stored.event.type === "subagent.updated" && stored.event.payload.id === taskId, |
| 1024 | ); |
| 1025 | const secondAckTask = secondAck.storedEvents.find( |
| 1026 | (stored) => |
| 1027 | stored.event.type === "subagent.updated" && stored.event.payload.id === taskId, |
| 1028 | ); |
| 1029 | assert.isDefined(firstAckTask); |
| 1030 | assert.isDefined(secondAckTask); |
| 1031 | if ( |
| 1032 | firstAckTask?.event.type !== "subagent.updated" || |
| 1033 | secondAckTask?.event.type !== "subagent.updated" |
| 1034 | ) { |
| 1035 | return yield* Effect.die(new Error("Acknowledge events missing.")); |
| 1036 | } |
| 1037 | assert.equal(firstAckTask.event.payload.completionDelivery?.state, "acknowledged"); |
| 1038 | assert.equal(secondAckTask.event.payload.completionDelivery?.state, "acknowledged"); |
| 1039 | assert.deepEqual( |
| 1040 | secondAckTask.event.payload.completionDelivery, |
| 1041 | firstAckTask.event.payload.completionDelivery, |
| 1042 | ); |
| 1043 | assert.deepEqual( |
| 1044 | secondAckTask.event.payload.updatedAt, |
| 1045 | firstAckTask.event.payload.updatedAt, |
| 1046 | ); |
| 1047 | assert.equal(secondAck.storedEvents.length, 1); |
| 1048 | |
| 1049 | const afterAck = yield* orchestrator.getThreadProjection(threadId); |
| 1050 | assert.deepEqual( |
| 1051 | afterAck.subagents.find((candidate) => candidate.id === taskId)?.completionDelivery, |
| 1052 | { |
| 1053 | state: "acknowledged", |
| 1054 | observedByRunId: runId, |
| 1055 | }, |
| 1056 | ); |
| 1057 | |
| 1058 | const firstDispose = yield* orchestrator.dispatch({ |
| 1059 | type: "delegated_task.completion-delivery.dispose", |
| 1060 | commandId: CommandId.make("command:delegated-delivery-a2:dispose-task-status"), |
| 1061 | parentThreadId: threadId, |
| 1062 | taskId, |
| 1063 | }); |
| 1064 | const secondDispose = yield* orchestrator.dispatch({ |
| 1065 | type: "delegated_task.completion-delivery.dispose", |
| 1066 | commandId: CommandId.make("command:delegated-delivery-a2:dispose-thread-read"), |
| 1067 | parentThreadId: threadId, |
| 1068 | taskId, |
| 1069 | }); |
| 1070 | |
| 1071 | const firstDisposeTask = firstDispose.storedEvents.find( |
| 1072 | (stored) => |
| 1073 | stored.event.type === "subagent.updated" && stored.event.payload.id === taskId, |
| 1074 | ); |
| 1075 | const secondDisposeTask = secondDispose.storedEvents.find( |
| 1076 | (stored) => |
| 1077 | stored.event.type === "subagent.updated" && stored.event.payload.id === taskId, |
| 1078 | ); |
| 1079 | assert.isDefined(firstDisposeTask); |
| 1080 | assert.isDefined(secondDisposeTask); |
| 1081 | if ( |
| 1082 | firstDisposeTask?.event.type !== "subagent.updated" || |
| 1083 | secondDisposeTask?.event.type !== "subagent.updated" |
| 1084 | ) { |
| 1085 | return yield* Effect.die(new Error("Dispose events missing.")); |
| 1086 | } |
| 1087 | assert.equal(firstDisposeTask.event.payload.completionDelivery?.state, "disposed"); |
| 1088 | assert.equal(secondDisposeTask.event.payload.completionDelivery?.state, "disposed"); |
| 1089 | assert.deepEqual( |
| 1090 | secondDisposeTask.event.payload.completionDelivery, |
| 1091 | firstDisposeTask.event.payload.completionDelivery, |
| 1092 | ); |
| 1093 | assert.equal(secondDispose.storedEvents.length, 1); |
| 1094 | |
| 1095 | const afterDispose = yield* orchestrator.getThreadProjection(threadId); |
| 1096 | assert.deepEqual( |
| 1097 | afterDispose.subagents.find((candidate) => candidate.id === taskId)?.completionDelivery, |
| 1098 | { |
| 1099 | state: "disposed", |
| 1100 | observedByRunId: null, |
| 1101 | }, |
| 1102 | ); |
| 1103 | |
| 1104 | const acknowledgeAfterDispose = yield* orchestrator.dispatch({ |
| 1105 | type: "delegated_task.completion-delivery.acknowledge", |
| 1106 | commandId: CommandId.make("command:delegated-delivery-a2:ack-after-dispose"), |
| 1107 | parentThreadId: threadId, |
| 1108 | taskId, |
| 1109 | observedByRunId: runId, |
| 1110 | }); |
| 1111 | const acknowledgedTask = acknowledgeAfterDispose.storedEvents.find( |
| 1112 | (stored) => stored.event.type === "subagent.updated", |
| 1113 | ); |
| 1114 | if (acknowledgedTask?.event.type !== "subagent.updated") { |
| 1115 | return yield* Effect.die(new Error("Acknowledge-after-dispose event missing.")); |
| 1116 | } |
| 1117 | assert.deepEqual(acknowledgedTask.event.payload.completionDelivery, { |
| 1118 | state: "disposed", |
| 1119 | observedByRunId: null, |
| 1120 | }); |
| 1121 | assert.equal(acknowledgeAfterDispose.storedEvents.length, 1); |
| 1122 | |
| 1123 | const afterStaleAcknowledge = yield* orchestrator.getThreadProjection(threadId); |
| 1124 | assert.deepEqual( |
| 1125 | afterStaleAcknowledge.subagents.find((candidate) => candidate.id === taskId) |
| 1126 | ?.completionDelivery, |
| 1127 | { |
| 1128 | state: "disposed", |
| 1129 | observedByRunId: null, |
| 1130 | }, |
| 1131 | ); |
| 1132 | }), |
| 1133 | ); |
| 1134 | |
| 1135 | it.effect("acknowledging a queued wake's results rewrites it, then cancels it", () => |
| 1136 | Effect.gen(function* () { |
| 1137 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 1138 | const now = yield* DateTime.now; |
| 1139 | const threadId = ThreadId.make("queued-wake-ack-parent"); |
| 1140 | const runId = RunId.make("queued-wake-ack-origin"); |
| 1141 | const taskId = NodeId.make("queued-wake-ack-first"); |
| 1142 | const siblingId = NodeId.make("queued-wake-ack-sibling"); |
| 1143 | yield* seedParentWithTerminalTask({ |
| 1144 | threadId, |
| 1145 | projectId: ProjectId.make("queued-wake-ack-project"), |
| 1146 | runId, |
| 1147 | rootNodeId: NodeId.make("queued-wake-ack-root"), |
| 1148 | taskId, |
| 1149 | deliveryState: "claimed", |
| 1150 | deliveryTaskIds: [taskId, siblingId], |
| 1151 | siblingTasks: [{ id: siblingId, deliveryState: "claimed" }], |
| 1152 | completionWake: "always", |
| 1153 | now, |
| 1154 | }); |
| 1155 | const delivery = (yield* orchestrator.getThreadProjection(threadId)).runs[0]! |
| 1156 | .delegatedCompletion!.delivery!; |
| 1157 | yield* orchestrator.dispatch({ |
| 1158 | type: "message.dispatch", |
| 1159 | createdBy: "agent", |
| 1160 | creationSource: "server", |
| 1161 | commandId: CommandId.make("queued-wake-ack-wake"), |
| 1162 | threadId, |
| 1163 | messageId: delivery.messageId, |
| 1164 | text: "Delegated tasks completed", |
| 1165 | attachments: [], |
| 1166 | modelSelection, |
| 1167 | dispatchMode: { type: "queue_after_active" }, |
| 1168 | delegatedCompletion: { |
| 1169 | parentRunId: runId, |
| 1170 | generation: delivery.generation, |
| 1171 | taskIds: delivery.taskIds, |
| 1172 | }, |
| 1173 | }); |
| 1174 | const acknowledge = (id: NodeId) => |
| 1175 | orchestrator.dispatch({ |
| 1176 | type: "delegated_task.completion-delivery.acknowledge", |
| 1177 | commandId: CommandId.make(`queued-wake-ack:${id}`), |
| 1178 | parentThreadId: threadId, |
| 1179 | taskId: id, |
| 1180 | observedByRunId: runId, |
| 1181 | }); |
| 1182 | |
| 1183 | yield* acknowledge(taskId); |
| 1184 | const rewritten = yield* orchestrator.getThreadProjection(threadId); |
| 1185 | const wake = rewritten.runs.find((run) => run.userMessageId === delivery.messageId)!; |
| 1186 | assert.equal(wake.status, "queued"); |
| 1187 | const message = rewritten.messages.find((row) => row.id === delivery.messageId); |
| 1188 | assert.deepEqual(message?.delegatedCompletion?.taskIds, [siblingId]); |
| 1189 | assert.equal(message?.text, `Delegated task ${siblingId} [completed]:\nchild finished`); |
| 1190 | |
| 1191 | yield* acknowledge(siblingId); |
| 1192 | const cancelled = yield* orchestrator.getThreadProjection(threadId); |
| 1193 | assert.equal(cancelled.runs.find((run) => run.id === wake.id)?.status, "cancelled"); |
| 1194 | assert.equal( |
| 1195 | cancelled.nodes.find((node) => node.id === wake.rootNodeId)?.status, |
| 1196 | "cancelled", |
| 1197 | ); |
| 1198 | assert.isNull(cancelled.runs.find((run) => run.id === runId)?.delegatedCompletion?.delivery); |
| 1199 | }), |
| 1200 | ); |
| 1201 | |
| 1202 | it.effect.each([ |
| 1203 | { command: "acknowledge", before: "running", after: "failed", capture: false }, |
| 1204 | { command: "acknowledge", before: "waiting", after: "completed", capture: true }, |
| 1205 | { command: "accept", before: "waiting", after: "completed", capture: true }, |
| 1206 | { command: "wake-policy", before: "running", after: "completed", capture: true }, |
| 1207 | ] as const)( |
| 1208 | "keeps a $after run that settled between $command's read and commit", |
| 1209 | ({ command, before, after, capture }) => |
| 1210 | Effect.gen(function* () { |
| 1211 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 1212 | const sink = yield* EventSink.EventSinkV2; |
| 1213 | const now = yield* DateTime.now; |
| 1214 | const race = `${command}-settle-race-${after}`; |
| 1215 | const threadId = ThreadId.make(`thread:${race}`); |
| 1216 | const runId = RunId.make(`run:${race}`); |
| 1217 | const taskId = NodeId.make(`node:${race}-task`); |
| 1218 | const checkpointId = CheckpointId.make(`checkpoint:${race}`); |
| 1219 | const messageId = MessageId.make(`message:delegated-delivery:${threadId}`); |
| 1220 | const commandType = { |
| 1221 | acknowledge: "delegated_task.completion-delivery.acknowledge", |
| 1222 | accept: "notification.delivery.accept", |
| 1223 | "wake-policy": "delegated_task.wake-policy", |
| 1224 | }[command]; |
| 1225 | yield* seedParentWithTerminalTask({ |
| 1226 | threadId, |
| 1227 | projectId: ProjectId.make(`project:${race}`), |
| 1228 | runId, |
| 1229 | rootNodeId: NodeId.make(`node:${race}-root`), |
| 1230 | taskId, |
| 1231 | ...(command === "wake-policy" |
| 1232 | ? { deliveryState: "pending" } |
| 1233 | : { deliveryState: "claimed", deliveryTaskIds: [taskId] }), |
| 1234 | now, |
| 1235 | }); |
| 1236 | const parent = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!; |
| 1237 | yield* sink.write({ |
| 1238 | events: [ |
| 1239 | { |
| 1240 | id: EventId.make(`event:${race}-before`), |
| 1241 | type: "run.updated", |
| 1242 | threadId, |
| 1243 | runId, |
| 1244 | occurredAt: now, |
| 1245 | payload: { ...parent, status: before }, |
| 1246 | }, |
| 1247 | { |
| 1248 | id: EventId.make(`event:${race}-message`), |
| 1249 | type: "message.updated", |
| 1250 | threadId, |
| 1251 | runId, |
| 1252 | occurredAt: now, |
| 1253 | payload: { |
| 1254 | id: messageId, |
| 1255 | threadId, |
| 1256 | runId, |
| 1257 | nodeId: parent.rootNodeId, |
| 1258 | role: "user", |
| 1259 | text: "Background task finished", |
| 1260 | attachments: [], |
| 1261 | streaming: false, |
| 1262 | createdBy: "agent", |
| 1263 | creationSource: "server", |
| 1264 | createdAt: now, |
| 1265 | updatedAt: now, |
| 1266 | delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [taskId] }, |
| 1267 | }, |
| 1268 | }, |
| 1269 | ], |
| 1270 | }); |
| 1271 | const { delegatedCompletion: _cohort, ...settledRun } = parent; |
| 1272 | const commit = sink.commitCommand; |
| 1273 | let settled = false; |
| 1274 | let planned: OrchestrationV2Run["delegatedCompletion"]; |
| 1275 | yield* Effect.acquireUseRelease( |
| 1276 | Effect.sync(() => |
| 1277 | vi.spyOn(sink, "commitCommand").mockImplementation((input) => |
| 1278 | Effect.gen(function* () { |
| 1279 | if (!settled && input.commandType === commandType) { |
| 1280 | settled = true; |
| 1281 | const plannedRun = input.events.find((event) => event.type === "run.updated"); |
| 1282 | planned = |
| 1283 | plannedRun?.type === "run.updated" |
| 1284 | ? plannedRun.payload.delegatedCompletion |
| 1285 | : undefined; |
| 1286 | // The provider turn ends (or its checkpoint lands) after the command read the run. |
| 1287 | yield* commit({ |
| 1288 | commandId: CommandId.make(`command:${race}`), |
| 1289 | threadId, |
| 1290 | commandType: capture ? "checkpoint.capture" : "provider.event", |
| 1291 | acceptedAt: now, |
| 1292 | effects: [], |
| 1293 | events: [ |
| 1294 | { |
| 1295 | id: EventId.make(`event:${race}-settled`), |
| 1296 | type: "run.updated", |
| 1297 | threadId, |
| 1298 | runId, |
| 1299 | occurredAt: now, |
| 1300 | payload: { |
| 1301 | ...settledRun, |
| 1302 | status: after, |
| 1303 | completedAt: now, |
| 1304 | checkpointId: capture ? checkpointId : null, |
| 1305 | }, |
| 1306 | }, |
| 1307 | ], |
| 1308 | }); |
| 1309 | } |
| 1310 | return yield* commit(input); |
| 1311 | }), |
| 1312 | ), |
| 1313 | ), |
| 1314 | () => |
| 1315 | orchestrator.dispatch( |
| 1316 | command === "accept" |
| 1317 | ? { |
| 1318 | type: "notification.delivery.accept", |
| 1319 | commandId: CommandId.make(`command:${race}:accept`), |
| 1320 | threadId, |
| 1321 | messageId, |
| 1322 | } |
| 1323 | : command === "wake-policy" |
| 1324 | ? { |
| 1325 | type: "delegated_task.wake-policy", |
| 1326 | commandId: CommandId.make(`command:${race}:wake-policy`), |
| 1327 | parentThreadId: threadId, |
| 1328 | taskId, |
| 1329 | completionWake: "always", |
| 1330 | } |
| 1331 | : { |
| 1332 | type: "delegated_task.completion-delivery.acknowledge", |
| 1333 | commandId: CommandId.make(`command:${race}:task-status`), |
| 1334 | parentThreadId: threadId, |
| 1335 | taskId, |
| 1336 | observedByRunId: runId, |
| 1337 | }, |
| 1338 | ), |
| 1339 | (spy) => Effect.sync(() => spy.mockRestore()), |
| 1340 | ); |
| 1341 | assert.isTrue(settled); |
| 1342 | const finished = yield* orchestrator.getThreadProjection(threadId); |
| 1343 | assert.equal( |
| 1344 | finished.subagents.find((row) => row.id === taskId)?.completionDelivery?.state, |
| 1345 | { acknowledge: "acknowledged", accept: "delivered", "wake-policy": "claimed" }[command], |
| 1346 | ); |
| 1347 | const run = finished.runs.find((candidate) => candidate.id === runId)!; |
| 1348 | assert.deepEqual( |
| 1349 | { |
| 1350 | status: run.status, |
| 1351 | completedAt: run.completedAt, |
| 1352 | checkpointId: run.checkpointId, |
| 1353 | delegatedCompletion: run.delegatedCompletion, |
| 1354 | }, |
| 1355 | { |
| 1356 | status: after, |
| 1357 | completedAt: now, |
| 1358 | checkpointId: capture ? checkpointId : null, |
| 1359 | delegatedCompletion: planned, |
| 1360 | }, |
| 1361 | ); |
| 1362 | }), |
| 1363 | ); |
| 1364 | }); |
| 1365 | |
| 1366 | const reconcileCommandId = (name: string) => CommandId.make(`command:runtime-reconcile:${name}`); |
| 1367 | |
| 1368 | const parentProviderThreadId = (threadId: ThreadId) => |
| 1369 | ProviderThreadId.make(`provider-thread:${String(threadId).replace("thread:", "")}`); |
| 1370 | |
| 1371 | const runEvent = (input: { |
| 1372 | readonly threadId: ThreadId; |
| 1373 | readonly runId: RunId; |
| 1374 | readonly ordinal: number; |
| 1375 | readonly status: OrchestrationV2Run["status"]; |
| 1376 | readonly now: DateTime.Utc; |
| 1377 | readonly providerThreadId?: ProviderThreadId; |
| 1378 | readonly userMessageId?: MessageId; |
| 1379 | readonly delegatedCompletion?: OrchestrationV2Run["delegatedCompletion"]; |
| 1380 | }) => ({ |
| 1381 | id: EventId.make(`event:${input.runId}:${input.status}`), |
| 1382 | type: "run.updated" as const, |
| 1383 | threadId: input.threadId, |
| 1384 | runId: input.runId, |
| 1385 | providerInstanceId: modelSelection.instanceId, |
| 1386 | occurredAt: input.now, |
| 1387 | payload: { |
| 1388 | id: input.runId, |
| 1389 | threadId: input.threadId, |
| 1390 | ordinal: input.ordinal, |
| 1391 | providerInstanceId: modelSelection.instanceId, |
| 1392 | modelSelection, |
| 1393 | providerThreadId: input.providerThreadId ?? null, |
| 1394 | userMessageId: input.userMessageId ?? MessageId.make(`message:${input.runId}`), |
| 1395 | rootNodeId: null, |
| 1396 | activeAttemptId: null, |
| 1397 | status: input.status, |
| 1398 | requestedAt: input.now, |
| 1399 | startedAt: input.now, |
| 1400 | completedAt: input.status === "running" ? null : input.now, |
| 1401 | checkpointId: null, |
| 1402 | contextHandoffId: null, |
| 1403 | ...(input.delegatedCompletion === undefined |
| 1404 | ? {} |
| 1405 | : { delegatedCompletion: input.delegatedCompletion }), |
| 1406 | }, |
| 1407 | }); |
| 1408 | |
| 1409 | const seedRestartCancelledChild = (input: { |
| 1410 | readonly parentThreadId: ThreadId; |
| 1411 | readonly projectId: ProjectId; |
| 1412 | readonly parentRunId: RunId; |
| 1413 | readonly rootNodeId: NodeId; |
| 1414 | readonly name: string; |
| 1415 | readonly completionWake: "always" | "settled_only"; |
| 1416 | readonly continuationPending: boolean; |
| 1417 | readonly runStatus?: "cancelled" | "completed"; |
| 1418 | readonly lostWork?: OrchestrationV2Run["restartCancelledBackgroundWork"]; |
| 1419 | readonly now: DateTime.Utc; |
| 1420 | }) => |
| 1421 | Effect.gen(function* () { |
| 1422 | const eventSink = yield* EventSink.EventSinkV2; |
| 1423 | const taskId = NodeId.make(`node:${input.name}`); |
| 1424 | const childThreadId = ThreadId.make(`thread:${input.name}`); |
| 1425 | const childRunId = RunId.make(`run:${input.name}:1`); |
| 1426 | yield* eventSink.write({ |
| 1427 | commandId: CommandId.make(`command:seed-child:${input.name}`), |
| 1428 | events: [ |
| 1429 | { |
| 1430 | id: EventId.make(`event:${input.name}:thread`), |
| 1431 | type: "thread.created", |
| 1432 | threadId: childThreadId, |
| 1433 | occurredAt: input.now, |
| 1434 | payload: { |
| 1435 | createdBy: "agent", |
| 1436 | creationSource: "server", |
| 1437 | id: childThreadId, |
| 1438 | projectId: input.projectId, |
| 1439 | title: input.name, |
| 1440 | providerInstanceId: modelSelection.instanceId, |
| 1441 | modelSelection, |
| 1442 | runtimeMode: "full-access", |
| 1443 | interactionMode: "default", |
| 1444 | branch: null, |
| 1445 | worktreePath: null, |
| 1446 | activeProviderThreadId: null, |
| 1447 | lineage: { |
| 1448 | parentThreadId: input.parentThreadId, |
| 1449 | relationshipToParent: "subagent", |
| 1450 | rootThreadId: input.parentThreadId, |
| 1451 | }, |
| 1452 | forkedFrom: { type: "node", nodeId: taskId }, |
| 1453 | createdAt: input.now, |
| 1454 | updatedAt: input.now, |
| 1455 | archivedAt: null, |
| 1456 | settledOverride: null, |
| 1457 | settledAt: null, |
| 1458 | lastVisitedAt: null, |
| 1459 | deletedAt: null, |
| 1460 | }, |
| 1461 | }, |
| 1462 | { |
| 1463 | id: EventId.make(`event:${input.name}:task`), |
| 1464 | type: "subagent.updated", |
| 1465 | threadId: input.parentThreadId, |
| 1466 | runId: input.parentRunId, |
| 1467 | nodeId: taskId, |
| 1468 | driver, |
| 1469 | providerInstanceId: modelSelection.instanceId, |
| 1470 | occurredAt: input.now, |
| 1471 | payload: { |
| 1472 | id: taskId, |
| 1473 | threadId: input.parentThreadId, |
| 1474 | runId: input.parentRunId, |
| 1475 | parentNodeId: input.rootNodeId, |
| 1476 | origin: "app_owned", |
| 1477 | createdBy: "agent", |
| 1478 | driver, |
| 1479 | providerInstanceId: modelSelection.instanceId, |
| 1480 | providerThreadId: null, |
| 1481 | childThreadId, |
| 1482 | nativeTaskRef: null, |
| 1483 | prompt: `Run ${input.name}.`, |
| 1484 | title: null, |
| 1485 | model: null, |
| 1486 | completionWake: input.completionWake, |
| 1487 | status: "running", |
| 1488 | result: null, |
| 1489 | startedAt: input.now, |
| 1490 | completedAt: null, |
| 1491 | updatedAt: input.now, |
| 1492 | }, |
| 1493 | }, |
| 1494 | ], |
| 1495 | }); |
| 1496 | const childRun = runEvent({ |
| 1497 | threadId: childThreadId, |
| 1498 | runId: childRunId, |
| 1499 | ordinal: 1, |
| 1500 | status: input.runStatus ?? "cancelled", |
| 1501 | now: input.now, |
| 1502 | }); |
| 1503 | yield* eventSink.writeWithEffects({ |
| 1504 | commandId: reconcileCommandId(input.name), |
| 1505 | events: [ |
| 1506 | input.lostWork === undefined |
| 1507 | ? childRun |
| 1508 | : { |
| 1509 | ...childRun, |
| 1510 | payload: { ...childRun.payload, restartCancelledBackgroundWork: input.lostWork }, |
| 1511 | }, |
| 1512 | ], |
| 1513 | effects: input.continuationPending |
| 1514 | ? [ |
| 1515 | { |
| 1516 | id: `effect:restart-continuation:${childRunId}`, |
| 1517 | commandId: reconcileCommandId(input.name), |
| 1518 | threadId: childThreadId, |
| 1519 | request: { type: "provider-runtime.continue", sourceRunId: childRunId }, |
| 1520 | }, |
| 1521 | ] |
| 1522 | : [], |
| 1523 | }); |
| 1524 | return { taskId, childThreadId, childRunId }; |
| 1525 | }); |
| 1526 | |
| 1527 | // Parent spawned two always-wake tasks: the incoming one's completion wake is queued behind a busy |
| 1528 | // run while the sibling's child run is still going. |
| 1529 | const queueCompletionWakeBesideRunningSibling = (key: string) => |
| 1530 | Effect.gen(function* () { |
| 1531 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 1532 | const sink = yield* EventSink.EventSinkV2; |
| 1533 | const now = yield* DateTime.now; |
| 1534 | const threadId = ThreadId.make(`${key}-parent`); |
| 1535 | const originRunId = RunId.make(`${key}-origin`); |
| 1536 | const originRootId = NodeId.make(`${key}-origin-root`); |
| 1537 | const incomingTaskId = NodeId.make(`${key}-incoming`); |
| 1538 | yield* seedParentWithTerminalTask({ |
| 1539 | threadId, |
| 1540 | projectId: ProjectId.make(`${key}-project`), |
| 1541 | runId: originRunId, |
| 1542 | rootNodeId: originRootId, |
| 1543 | taskId: incomingTaskId, |
| 1544 | deliveryState: "claimed", |
| 1545 | deliveryTaskIds: [incomingTaskId], |
| 1546 | completionWake: "always", |
| 1547 | now, |
| 1548 | }); |
| 1549 | yield* sink.write({ |
| 1550 | events: [ |
| 1551 | { |
| 1552 | id: EventId.make(`event:${key}-origin-root`), |
| 1553 | type: "node.updated", |
| 1554 | threadId, |
| 1555 | runId: originRunId, |
| 1556 | nodeId: originRootId, |
| 1557 | occurredAt: now, |
| 1558 | payload: { |
| 1559 | id: originRootId, |
| 1560 | threadId, |
| 1561 | runId: originRunId, |
| 1562 | parentNodeId: null, |
| 1563 | rootNodeId: originRootId, |
| 1564 | kind: "root_turn", |
| 1565 | status: "running", |
| 1566 | countsForRun: true, |
| 1567 | providerThreadId: null, |
| 1568 | providerTurnId: null, |
| 1569 | nativeItemRef: null, |
| 1570 | runtimeRequestId: null, |
| 1571 | checkpointScopeId: null, |
| 1572 | startedAt: now, |
| 1573 | completedAt: null, |
| 1574 | }, |
| 1575 | }, |
| 1576 | ], |
| 1577 | }); |
| 1578 | yield* orchestrator.dispatch({ |
| 1579 | type: "delegated_task.request", |
| 1580 | createdBy: "agent", |
| 1581 | creationSource: "mcp", |
| 1582 | commandId: CommandId.make(`${key}-sibling`), |
| 1583 | parentThreadId: threadId, |
| 1584 | parentRunId: originRunId, |
| 1585 | parentNodeId: originRootId, |
| 1586 | task: "Keep working", |
| 1587 | modelSelection, |
| 1588 | runtimeMode: "full-access", |
| 1589 | interactionMode: "default", |
| 1590 | completionWake: "always", |
| 1591 | }); |
| 1592 | const seeded = yield* orchestrator.getThreadProjection(threadId); |
| 1593 | const origin = seeded.runs.find((run) => run.id === originRunId)!; |
| 1594 | const sibling = seeded.subagents.find((task) => task.id !== incomingTaskId)!; |
| 1595 | // The coordinator finished its spawning turn but is busy with another run, so the wake queues. |
| 1596 | yield* sink.write({ |
| 1597 | events: [ |
| 1598 | { |
| 1599 | ...runEvent({ threadId, runId: originRunId, ordinal: 1, status: "completed", now }), |
| 1600 | payload: { ...origin, status: "completed", completedAt: now }, |
| 1601 | }, |
| 1602 | runEvent({ |
| 1603 | threadId, |
| 1604 | runId: RunId.make(`${key}-busy`), |
| 1605 | ordinal: 2, |
| 1606 | status: "running", |
| 1607 | now, |
| 1608 | }), |
| 1609 | ], |
| 1610 | }); |
| 1611 | const delivery = origin.delegatedCompletion!.delivery!; |
| 1612 | yield* orchestrator.dispatch({ |
| 1613 | type: "message.dispatch", |
| 1614 | createdBy: "agent", |
| 1615 | creationSource: "server", |
| 1616 | commandId: CommandId.make(`${key}-wake`), |
| 1617 | threadId, |
| 1618 | messageId: delivery.messageId, |
| 1619 | text: "Delegated task completed", |
| 1620 | attachments: [], |
| 1621 | modelSelection, |
| 1622 | dispatchMode: { type: "queue_after_active" }, |
| 1623 | delegatedCompletion: { |
| 1624 | parentRunId: originRunId, |
| 1625 | generation: delivery.generation, |
| 1626 | taskIds: delivery.taskIds, |
| 1627 | }, |
| 1628 | }); |
| 1629 | const wake = (yield* orchestrator.getThreadProjection(threadId)).runs.find( |
| 1630 | (run) => run.userMessageId === delivery.messageId, |
| 1631 | )!; |
| 1632 | assert.equal(wake.status, "queued"); |
| 1633 | const completeSibling = Effect.gen(function* () { |
| 1634 | const childRun = (yield* orchestrator.getThreadProjection(sibling.childThreadId!)).runs[0]!; |
| 1635 | const afterSequence = yield* sink.latestSequence(); |
| 1636 | yield* sink.write({ |
| 1637 | events: [ |
| 1638 | { |
| 1639 | ...runEvent({ |
| 1640 | threadId: sibling.childThreadId!, |
| 1641 | runId: childRun.id, |
| 1642 | ordinal: childRun.ordinal, |
| 1643 | status: "completed", |
| 1644 | now, |
| 1645 | }), |
| 1646 | payload: { ...childRun, status: "completed", completedAt: now }, |
| 1647 | }, |
| 1648 | ], |
| 1649 | }); |
| 1650 | return afterSequence; |
| 1651 | }); |
| 1652 | return { |
| 1653 | orchestrator, |
| 1654 | sink, |
| 1655 | now, |
| 1656 | threadId, |
| 1657 | originRunId, |
| 1658 | incomingTaskId, |
| 1659 | sibling, |
| 1660 | delivery, |
| 1661 | wake, |
| 1662 | completeSibling, |
| 1663 | }; |
| 1664 | }); |
| 1665 | |
| 1666 | it.layer(layerContinuationWorkerTest)("interrupted completion wakes", (it) => { |
| 1667 | it.effect.each([ |
| 1668 | { name: "an interrupt keeps", holdQueue: false, siblingFinishesFirst: false }, |
| 1669 | { name: "Stop disposes", holdQueue: true, siblingFinishesFirst: false }, |
| 1670 | { name: "an interrupt reserves", holdQueue: false, siblingFinishesFirst: true }, |
| 1671 | ])("$name a completion wake's other siblings", ({ holdQueue, siblingFinishesFirst }) => |
| 1672 | Effect.gen(function* () { |
| 1673 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 1674 | const sink = yield* EventSink.EventSinkV2; |
| 1675 | const now = yield* DateTime.now; |
| 1676 | const key = holdQueue |
| 1677 | ? "sibling-stop" |
| 1678 | : siblingFinishesFirst |
| 1679 | ? "sibling-finished-interrupt" |
| 1680 | : "sibling-interrupt"; |
| 1681 | const threadId = ThreadId.make(`${key}-parent`); |
| 1682 | const originRunId = RunId.make(`${key}-origin`); |
| 1683 | const incomingTaskId = NodeId.make(`${key}-incoming`); |
| 1684 | yield* seedParentWithTerminalTask({ |
| 1685 | threadId, |
| 1686 | projectId: ProjectId.make(`${key}-project`), |
| 1687 | runId: originRunId, |
| 1688 | rootNodeId: NodeId.make(`${key}-origin-root`), |
| 1689 | taskId: incomingTaskId, |
| 1690 | deliveryState: "claimed", |
| 1691 | deliveryTaskIds: [incomingTaskId], |
| 1692 | completionWake: "always", |
| 1693 | now, |
| 1694 | }); |
| 1695 | const originRootId = NodeId.make(`${key}-origin-root`); |
| 1696 | yield* sink.write({ |
| 1697 | events: [ |
| 1698 | { |
| 1699 | id: EventId.make(`event:${key}-origin-root`), |
| 1700 | type: "node.updated", |
| 1701 | threadId, |
| 1702 | runId: originRunId, |
| 1703 | nodeId: originRootId, |
| 1704 | occurredAt: now, |
| 1705 | payload: { |
| 1706 | id: originRootId, |
| 1707 | threadId, |
| 1708 | runId: originRunId, |
| 1709 | parentNodeId: null, |
| 1710 | rootNodeId: originRootId, |
| 1711 | kind: "root_turn", |
| 1712 | status: "running", |
| 1713 | countsForRun: true, |
| 1714 | providerThreadId: null, |
| 1715 | providerTurnId: null, |
| 1716 | nativeItemRef: null, |
| 1717 | runtimeRequestId: null, |
| 1718 | checkpointScopeId: null, |
| 1719 | startedAt: now, |
| 1720 | completedAt: null, |
| 1721 | }, |
| 1722 | }, |
| 1723 | ], |
| 1724 | }); |
| 1725 | yield* orchestrator.dispatch({ |
| 1726 | type: "delegated_task.request", |
| 1727 | createdBy: "agent", |
| 1728 | creationSource: "mcp", |
| 1729 | commandId: CommandId.make(`${key}-sibling`), |
| 1730 | parentThreadId: threadId, |
| 1731 | parentRunId: originRunId, |
| 1732 | parentNodeId: originRootId, |
| 1733 | task: "Keep working", |
| 1734 | modelSelection, |
| 1735 | runtimeMode: "full-access", |
| 1736 | interactionMode: "default", |
| 1737 | completionWake: "always", |
| 1738 | }); |
| 1739 | const seeded = yield* orchestrator.getThreadProjection(threadId); |
| 1740 | const origin = seeded.runs.find((run) => run.id === originRunId)!; |
| 1741 | const sibling = seeded.subagents.find((task) => task.id !== incomingTaskId)!; |
| 1742 | yield* sink.write({ |
| 1743 | events: [ |
| 1744 | { |
| 1745 | ...runEvent({ threadId, runId: originRunId, ordinal: 1, status: "completed", now }), |
| 1746 | payload: { ...origin, status: "completed", completedAt: now }, |
| 1747 | }, |
| 1748 | ], |
| 1749 | }); |
| 1750 | const delivery = origin.delegatedCompletion!.delivery!; |
| 1751 | yield* orchestrator.dispatch({ |
| 1752 | type: "message.dispatch", |
| 1753 | createdBy: "agent", |
| 1754 | creationSource: "server", |
| 1755 | commandId: CommandId.make(`${key}-wake`), |
| 1756 | threadId, |
| 1757 | messageId: delivery.messageId, |
| 1758 | text: "Delegated task completed", |
| 1759 | attachments: [], |
| 1760 | modelSelection, |
| 1761 | dispatchMode: { type: "queue_after_active" }, |
| 1762 | delegatedCompletion: { |
| 1763 | parentRunId: originRunId, |
| 1764 | generation: delivery.generation, |
| 1765 | taskIds: delivery.taskIds, |
| 1766 | }, |
| 1767 | }); |
| 1768 | yield* orchestrator.resumeQueuedRuns; |
| 1769 | const wake = (yield* orchestrator.getThreadProjection(threadId)).runs.find( |
| 1770 | (run) => run.userMessageId === delivery.messageId, |
| 1771 | )!; |
| 1772 | const childRun = (yield* orchestrator.getThreadProjection(sibling.childThreadId!)).runs[0]!; |
| 1773 | const completeSibling = Effect.gen(function* () { |
| 1774 | const afterSequence = yield* sink.latestSequence(); |
| 1775 | yield* sink.write({ |
| 1776 | events: [ |
| 1777 | { |
| 1778 | ...runEvent({ |
| 1779 | threadId: sibling.childThreadId!, |
| 1780 | runId: childRun.id, |
| 1781 | ordinal: childRun.ordinal, |
| 1782 | status: "completed", |
| 1783 | now, |
| 1784 | }), |
| 1785 | payload: { ...childRun, status: "completed", completedAt: now }, |
| 1786 | }, |
| 1787 | ], |
| 1788 | }); |
| 1789 | return yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe( |
| 1790 | Stream.filter( |
| 1791 | (stored) => |
| 1792 | stored.event.type === "subagent.updated" && |
| 1793 | stored.event.payload.id === sibling.id && |
| 1794 | stored.event.payload.status === "completed", |
| 1795 | ), |
| 1796 | Stream.take(1), |
| 1797 | Stream.runDrain, |
| 1798 | Effect.as(afterSequence), |
| 1799 | ); |
| 1800 | }); |
| 1801 | if (siblingFinishesFirst) { |
| 1802 | yield* completeSibling; |
| 1803 | assert.equal( |
| 1804 | (yield* orchestrator.getThreadProjection(threadId)).subagents.find( |
| 1805 | (task) => task.id === sibling.id, |
| 1806 | )?.completionDelivery?.state, |
| 1807 | "pending", |
| 1808 | ); |
| 1809 | } |
| 1810 | const beforeInterrupt = yield* sink.latestSequence(); |
| 1811 | const interrupted = yield* orchestrator.dispatch({ |
| 1812 | type: "run.interrupt", |
| 1813 | commandId: CommandId.make(`${key}-interrupt`), |
| 1814 | threadId, |
| 1815 | runId: wake.id, |
| 1816 | ...(holdQueue ? { holdQueue } : {}), |
| 1817 | }); |
| 1818 | // The wake's terminal may claim the sibling right after commit, so read what the interrupt |
| 1819 | // itself wrote rather than a later projection. |
| 1820 | const interruptDelivery = new Map( |
| 1821 | interrupted.storedEvents.flatMap(({ event }) => |
| 1822 | event.type === "subagent.updated" |
| 1823 | ? [[event.payload.id, event.payload.completionDelivery?.state] as const] |
| 1824 | : [], |
| 1825 | ), |
| 1826 | ); |
| 1827 | const interruptCohort = interrupted.storedEvents.flatMap(({ event }) => |
| 1828 | event.type === "run.updated" && event.payload.id === originRunId |
| 1829 | ? [event.payload.delegatedCompletion] |
| 1830 | : [], |
| 1831 | ); |
| 1832 | assert.equal(interruptDelivery.get(incomingTaskId), "disposed"); |
| 1833 | assert.equal(interruptCohort.at(-1)?.disposition, holdQueue ? "stopped" : "open"); |
| 1834 | if (holdQueue) { |
| 1835 | assert.equal(interruptDelivery.get(sibling.id), "disposed"); |
| 1836 | return; |
| 1837 | } |
| 1838 | assert.isFalse(interruptDelivery.has(sibling.id)); |
| 1839 | |
| 1840 | const afterSequence = siblingFinishesFirst ? beforeInterrupt : yield* completeSibling; |
| 1841 | // The wake's terminal, or the sibling's own completion, reserves the successor. |
| 1842 | yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe( |
| 1843 | Stream.filter( |
| 1844 | (stored) => |
| 1845 | stored.event.type === "subagent.updated" && |
| 1846 | stored.event.payload.id === sibling.id && |
| 1847 | stored.event.payload.completionDelivery?.state === "claimed", |
| 1848 | ), |
| 1849 | Stream.take(1), |
| 1850 | Stream.runDrain, |
| 1851 | ); |
| 1852 | const woken = yield* orchestrator.getThreadProjection(threadId); |
| 1853 | const successor = woken.runs.find((run) => run.id === originRunId)?.delegatedCompletion |
| 1854 | ?.delivery; |
| 1855 | assert.equal(successor?.generation, delivery.generation + 1); |
| 1856 | assert.deepEqual(successor?.taskIds, [sibling.id]); |
| 1857 | assert.notEqual(successor?.messageId, delivery.messageId); |
| 1858 | |
| 1859 | // ProviderContinuationService consumes the claim's offer and dispatches the successor. |
| 1860 | yield* sink.stream({ afterSequence, eventType: "message.updated" }).pipe( |
| 1861 | Stream.filter( |
| 1862 | (stored) => |
| 1863 | stored.event.type === "message.updated" && |
| 1864 | stored.event.payload.id === successor!.messageId, |
| 1865 | ), |
| 1866 | Stream.take(1), |
| 1867 | Stream.runDrain, |
| 1868 | ); |
| 1869 | // Repeated recovery re-offers every open delivery; none may wake the parent twice. |
| 1870 | yield* orchestrator.resumeQueuedRuns; |
| 1871 | yield* orchestrator.recoverDelegatedTasks; |
| 1872 | yield* orchestrator.recoverDelegatedTasks; |
| 1873 | yield* orchestrator.resumeQueuedRuns; |
| 1874 | const delivered = yield* orchestrator.getThreadProjection(threadId); |
| 1875 | const cohort = delivered.runs.find((run) => run.id === originRunId)?.delegatedCompletion; |
| 1876 | assert.equal(cohort?.disposition, "open"); |
| 1877 | assert.equal(cohort?.nextGeneration, delivery.generation + 2); |
| 1878 | assert.deepEqual(cohort?.delivery, successor); |
| 1879 | assert.deepEqual( |
| 1880 | delivered.messages |
| 1881 | .filter((message) => message.delegatedCompletion?.parentRunId === originRunId) |
| 1882 | .map((message) => [message.id, message.delegatedCompletion]), |
| 1883 | [ |
| 1884 | [ |
| 1885 | delivery.messageId, |
| 1886 | { |
| 1887 | parentRunId: originRunId, |
| 1888 | generation: delivery.generation, |
| 1889 | taskIds: [incomingTaskId], |
| 1890 | }, |
| 1891 | ], |
| 1892 | [ |
| 1893 | successor!.messageId, |
| 1894 | { parentRunId: originRunId, generation: successor!.generation, taskIds: [sibling.id] }, |
| 1895 | ], |
| 1896 | ], |
| 1897 | ); |
| 1898 | const successorRuns = delivered.runs.filter( |
| 1899 | (run) => run.userMessageId === successor!.messageId, |
| 1900 | ); |
| 1901 | assert.equal(successorRuns.length, 1); |
| 1902 | assert.equal(successorRuns[0]?.status, "starting"); |
| 1903 | assert.equal( |
| 1904 | delivered.subagents.find((task) => task.id === sibling.id)?.completionDelivery?.state, |
| 1905 | "claimed", |
| 1906 | ); |
| 1907 | }), |
| 1908 | ); |
| 1909 | |
| 1910 | it.effect("cancelling a queued completion wake keeps its other siblings", () => |
| 1911 | Effect.gen(function* () { |
| 1912 | const { |
| 1913 | orchestrator, |
| 1914 | sink, |
| 1915 | threadId, |
| 1916 | originRunId, |
| 1917 | incomingTaskId, |
| 1918 | sibling, |
| 1919 | delivery, |
| 1920 | wake, |
| 1921 | completeSibling, |
| 1922 | } = yield* queueCompletionWakeBesideRunningSibling("sibling-queued-cancel"); |
| 1923 | const cancelled = yield* orchestrator.dispatch({ |
| 1924 | type: "queued-run.cancel", |
| 1925 | commandId: CommandId.make("sibling-queued-cancel-cancel"), |
| 1926 | threadId, |
| 1927 | runId: wake.id, |
| 1928 | }); |
| 1929 | const cancelDelivery = new Map( |
| 1930 | cancelled.storedEvents.flatMap(({ event }) => |
| 1931 | event.type === "subagent.updated" |
| 1932 | ? [[event.payload.id, event.payload.completionDelivery?.state] as const] |
| 1933 | : [], |
| 1934 | ), |
| 1935 | ); |
| 1936 | assert.equal(cancelDelivery.get(incomingTaskId), "disposed"); |
| 1937 | assert.notEqual( |
| 1938 | cancelDelivery.get(sibling.id), |
| 1939 | "disposed", |
| 1940 | "cancelling the wake must not dispose a still-running sibling it did not carry", |
| 1941 | ); |
| 1942 | const afterCancel = yield* orchestrator.getThreadProjection(threadId); |
| 1943 | assert.equal( |
| 1944 | afterCancel.runs.find((run) => run.id === originRunId)?.delegatedCompletion?.disposition, |
| 1945 | "open", |
| 1946 | ); |
| 1947 | |
| 1948 | const afterSequence = yield* completeSibling; |
| 1949 | yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe( |
| 1950 | Stream.filter( |
| 1951 | (stored) => |
| 1952 | stored.event.type === "subagent.updated" && |
| 1953 | stored.event.payload.id === sibling.id && |
| 1954 | stored.event.payload.completionDelivery?.state === "claimed", |
| 1955 | ), |
| 1956 | Stream.take(1), |
| 1957 | Stream.runDrain, |
| 1958 | ); |
| 1959 | const successor = (yield* orchestrator.getThreadProjection(threadId)).runs.find( |
| 1960 | (run) => run.id === originRunId, |
| 1961 | )?.delegatedCompletion?.delivery; |
| 1962 | assert.deepEqual(successor?.taskIds, [sibling.id]); |
| 1963 | assert.notEqual(successor?.messageId, delivery.messageId); |
| 1964 | }), |
| 1965 | ); |
| 1966 | |
| 1967 | it.effect("settling with a queued completion wake disposes its running siblings", () => |
| 1968 | Effect.gen(function* () { |
| 1969 | const { orchestrator, sink, now, threadId, originRunId, sibling, wake, completeSibling } = |
| 1970 | yield* queueCompletionWakeBesideRunningSibling("sibling-settle"); |
| 1971 | const busy = (yield* orchestrator.getThreadProjection(threadId)).runs.find( |
| 1972 | (run) => run.status === "running", |
| 1973 | )!; |
| 1974 | yield* sink.write({ |
| 1975 | events: [ |
| 1976 | { |
| 1977 | ...runEvent({ |
| 1978 | threadId, |
| 1979 | runId: busy.id, |
| 1980 | ordinal: busy.ordinal, |
| 1981 | status: "cancelled", |
| 1982 | now, |
| 1983 | }), |
| 1984 | payload: { ...busy, status: "cancelled", completedAt: now }, |
| 1985 | }, |
| 1986 | { |
| 1987 | ...runEvent({ threadId, runId: wake.id, ordinal: wake.ordinal, status: "queued", now }), |
| 1988 | payload: { ...wake, queueHeld: true }, |
| 1989 | }, |
| 1990 | ], |
| 1991 | }); |
| 1992 | |
| 1993 | const settled = yield* orchestrator.dispatch({ |
| 1994 | type: "thread.settle", |
| 1995 | commandId: CommandId.make("sibling-settle-settle"), |
| 1996 | threadId, |
| 1997 | }); |
| 1998 | assert.deepEqual( |
| 1999 | settled.storedEvents.flatMap(({ event }) => |
| 2000 | event.type === "run.updated" && event.payload.id === wake.id |
| 2001 | ? [event.payload.status] |
| 2002 | : [], |
| 2003 | ), |
| 2004 | ["cancelled"], |
| 2005 | ); |
| 2006 | const afterSettle = yield* orchestrator.getThreadProjection(threadId); |
| 2007 | assert.equal( |
| 2008 | afterSettle.subagents.find((task) => task.id === sibling.id)?.completionDelivery?.state, |
| 2009 | "disposed", |
| 2010 | ); |
| 2011 | |
| 2012 | const afterSequence = yield* completeSibling; |
| 2013 | yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe( |
| 2014 | Stream.filter( |
| 2015 | (stored) => |
| 2016 | stored.event.type === "subagent.updated" && |
| 2017 | stored.event.payload.id === sibling.id && |
| 2018 | stored.event.payload.status === "completed", |
| 2019 | ), |
| 2020 | Stream.take(1), |
| 2021 | Stream.runDrain, |
| 2022 | ); |
| 2023 | const afterLateResult = yield* orchestrator.getThreadProjection(threadId); |
| 2024 | assert.equal( |
| 2025 | afterLateResult.subagents.find((task) => task.id === sibling.id)?.completionDelivery?.state, |
| 2026 | "disposed", |
| 2027 | ); |
| 2028 | assert.deepEqual( |
| 2029 | afterLateResult.runs.map((run) => run.id), |
| 2030 | afterSettle.runs.map((run) => run.id), |
| 2031 | ); |
| 2032 | assert.equal( |
| 2033 | afterLateResult.runs.find((run) => run.id === originRunId)?.delegatedCompletion |
| 2034 | ?.disposition, |
| 2035 | "disposed", |
| 2036 | ); |
| 2037 | assert.equal(afterLateResult.thread.settledOverride, "settled"); |
| 2038 | }), |
| 2039 | ); |
| 2040 | }); |
| 2041 | |
| 2042 | it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( |
| 2043 | "subagent follow-ups", |
| 2044 | (it) => { |
| 2045 | it.effect("a late unobserved read wakes a pending result without restarting", () => |
| 2046 | Effect.gen(function* () { |
| 2047 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 2048 | const sink = yield* EventSink.EventSinkV2; |
| 2049 | const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; |
| 2050 | const now = yield* DateTime.now; |
| 2051 | const threadId = ThreadId.make("thread:late-read-live"); |
| 2052 | const runId = RunId.make("run:late-read-live"); |
| 2053 | const taskId = NodeId.make("task:late-read-live"); |
| 2054 | yield* seedParentWithTerminalTask({ |
| 2055 | threadId, |
| 2056 | projectId: ProjectId.make("project:late-read-live"), |
| 2057 | runId, |
| 2058 | rootNodeId: NodeId.make("root:late-read-live"), |
| 2059 | taskId, |
| 2060 | deliveryState: "pending", |
| 2061 | now, |
| 2062 | }); |
| 2063 | const acknowledge = (key: string) => |
| 2064 | orchestrator.dispatch({ |
| 2065 | type: "delegated_task.completion-delivery.acknowledge", |
| 2066 | commandId: CommandId.make(`command:late-read-live:${key}`), |
| 2067 | parentThreadId: threadId, |
| 2068 | taskId, |
| 2069 | observedByRunId: null, |
| 2070 | }); |
| 2071 | const before = yield* orchestrator.getThreadProjection(threadId); |
| 2072 | const liveRead = yield* acknowledge("live"); |
| 2073 | assert.lengthOf(liveRead.storedEvents, 0); |
| 2074 | const parent = before.runs.find((run) => run.id === runId)!; |
| 2075 | yield* sink.write({ |
| 2076 | events: [ |
| 2077 | { |
| 2078 | id: EventId.make("event:late-read-live:parent-ended"), |
| 2079 | type: "run.updated", |
| 2080 | threadId, |
| 2081 | runId, |
| 2082 | occurredAt: now, |
| 2083 | payload: { ...parent, status: "completed", completedAt: now }, |
| 2084 | }, |
| 2085 | ], |
| 2086 | }); |
| 2087 | yield* acknowledge("ended"); |
| 2088 | const wake = yield* requests.take; |
| 2089 | assert.equal(wake.threadId, threadId); |
| 2090 | assert.equal(wake.delegatedCompletion?.parentRunId, runId); |
| 2091 | const after = yield* orchestrator.getThreadProjection(threadId); |
| 2092 | assert.equal( |
| 2093 | after.subagents.find((task) => task.id === taskId)?.completionDelivery?.state, |
| 2094 | "claimed", |
| 2095 | ); |
| 2096 | assert.deepEqual( |
| 2097 | after.runs.find((run) => run.id === runId)?.delegatedCompletion?.delivery?.taskIds, |
| 2098 | [taskId], |
| 2099 | ); |
| 2100 | const repeated = yield* acknowledge("repeated"); |
| 2101 | assert.lengthOf(repeated.storedEvents, 0); |
| 2102 | assert.deepEqual(yield* orchestrator.getThreadProjection(threadId), after); |
| 2103 | yield* orchestrator.dispatch({ |
| 2104 | type: "delegated_task.completion-delivery.dispose", |
| 2105 | commandId: CommandId.make("command:late-read-live:dispose"), |
| 2106 | parentThreadId: threadId, |
| 2107 | taskId, |
| 2108 | }); |
| 2109 | }), |
| 2110 | ); |
| 2111 | |
| 2112 | it.effect("recovers a reserved delivery without loading conversation history", () => |
| 2113 | Effect.gen(function* () { |
| 2114 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 2115 | const sink = yield* EventSink.EventSinkV2; |
| 2116 | const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; |
| 2117 | const now = yield* DateTime.now; |
| 2118 | const threadId = ThreadId.make("thread:bounded-delivery-recovery"); |
| 2119 | const runId = RunId.make("run:bounded-delivery-recovery"); |
| 2120 | const taskId = NodeId.make("task:bounded-delivery-recovery"); |
| 2121 | yield* seedParentWithTerminalTask({ |
| 2122 | threadId, |
| 2123 | runId, |
| 2124 | taskId, |
| 2125 | projectId: ProjectId.make("project:bounded-delivery-recovery"), |
| 2126 | rootNodeId: NodeId.make("node:bounded-delivery-recovery"), |
| 2127 | deliveryState: "claimed", |
| 2128 | deliveryTaskIds: [taskId], |
| 2129 | completionWake: "always", |
| 2130 | now, |
| 2131 | }); |
| 2132 | for (const suffix of ["first", "second"]) { |
| 2133 | yield* sink.write({ |
| 2134 | events: [ |
| 2135 | { |
| 2136 | id: EventId.make(`event:bounded-delivery-history:${suffix}`), |
| 2137 | type: "message.updated", |
| 2138 | threadId, |
| 2139 | occurredAt: now, |
| 2140 | payload: { |
| 2141 | id: MessageId.make(`message:bounded-delivery-history:${suffix}`), |
| 2142 | threadId, |
| 2143 | runId: null, |
| 2144 | nodeId: null, |
| 2145 | createdBy: "user", |
| 2146 | creationSource: "web", |
| 2147 | role: "user", |
| 2148 | text: "Unrelated historical input", |
| 2149 | attachments: [], |
| 2150 | streaming: false, |
| 2151 | createdAt: now, |
| 2152 | updatedAt: now, |
| 2153 | }, |
| 2154 | }, |
| 2155 | ], |
| 2156 | }); |
| 2157 | } |
| 2158 | const statements: string[] = []; |
| 2159 | const tracer = Tracer.make({ |
| 2160 | span(options) { |
| 2161 | const span = new Tracer.NativeSpan(options); |
| 2162 | const end = span.end.bind(span); |
| 2163 | span.end = (time, exit) => { |
| 2164 | end(time, exit); |
| 2165 | const query = span.attributes.get("db.query.text"); |
| 2166 | if (typeof query === "string") statements.push(query); |
| 2167 | }; |
| 2168 | return span; |
| 2169 | }, |
| 2170 | }); |
| 2171 | yield* orchestrator.recoverDelegatedTasks.pipe(Effect.withTracer(tracer)); |
| 2172 | assert.isFalse(statements.some((query) => query.includes("MIN(ordinal)"))); |
| 2173 | const messageReads = statements.filter((query) => |
| 2174 | query.includes("FROM orchestration_v2_projection_messages"), |
| 2175 | ); |
| 2176 | assert.isNotEmpty(messageReads); |
| 2177 | assert.isTrue(messageReads.every((query) => query.includes("message_id IN"))); |
| 2178 | const wake = yield* requests.take; |
| 2179 | assert.equal(wake.delegatedCompletion?.parentRunId, runId); |
| 2180 | }), |
| 2181 | ); |
| 2182 | it.effect("keeps a checkpoint completion committed between finalize's read and write", () => |
| 2183 | Effect.gen(function* () { |
| 2184 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 2185 | const sink = yield* EventSink.EventSinkV2; |
| 2186 | const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; |
| 2187 | const now = yield* DateTime.now; |
| 2188 | const threadId = ThreadId.make("thread:completion-capture-race"); |
| 2189 | const projectId = ProjectId.make("project:completion-capture-race"); |
| 2190 | const runId = RunId.make("run:completion-capture-race"); |
| 2191 | const rootNodeId = NodeId.make("node:completion-capture-race"); |
| 2192 | const checkpointId = CheckpointId.make("checkpoint:completion-capture-race"); |
| 2193 | yield* seedParentWithTerminalTask({ |
| 2194 | threadId, |
| 2195 | projectId, |
| 2196 | runId, |
| 2197 | rootNodeId, |
| 2198 | taskId: NodeId.make("node:completion-capture-seed"), |
| 2199 | deliveryState: "acknowledged", |
| 2200 | now, |
| 2201 | }); |
| 2202 | const parent = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!; |
| 2203 | const waiting = { ...parent, status: "waiting" as const }; |
| 2204 | yield* sink.write({ |
| 2205 | events: [ |
| 2206 | { |
| 2207 | id: EventId.make("event:completion-capture-waiting"), |
| 2208 | type: "run.updated", |
| 2209 | threadId, |
| 2210 | runId, |
| 2211 | occurredAt: now, |
| 2212 | payload: waiting, |
| 2213 | }, |
| 2214 | ], |
| 2215 | }); |
| 2216 | const child = yield* seedRestartCancelledChild({ |
| 2217 | parentThreadId: threadId, |
| 2218 | projectId, |
| 2219 | parentRunId: runId, |
| 2220 | rootNodeId, |
| 2221 | name: "completion-capture-child", |
| 2222 | completionWake: "always", |
| 2223 | continuationPending: false, |
| 2224 | runStatus: "completed", |
| 2225 | now, |
| 2226 | }); |
| 2227 | const write = sink.writeWithEffects; |
| 2228 | let captured = false; |
| 2229 | const committedParentRuns: Array<OrchestrationV2Run> = []; |
| 2230 | yield* Effect.acquireUseRelease( |
| 2231 | Effect.sync(() => |
| 2232 | vi.spyOn(sink, "writeWithEffects").mockImplementation((input) => |
| 2233 | Effect.gen(function* () { |
| 2234 | if ( |
| 2235 | !captured && |
| 2236 | input.events.some( |
| 2237 | (event) => |
| 2238 | event.type === "run.updated" && |
| 2239 | event.payload.id === runId && |
| 2240 | event.payload.delegatedCompletion?.delivery != null, |
| 2241 | ) |
| 2242 | ) { |
| 2243 | captured = true; |
| 2244 | // Commit capture after finalize planned its stale waiting snapshot. |
| 2245 | const { delegatedCompletion: _cohort, ...captureRun } = waiting; |
| 2246 | yield* sink.commitCommand({ |
| 2247 | commandId: CommandId.make("command:completion-capture-race"), |
| 2248 | threadId, |
| 2249 | commandType: "checkpoint.capture", |
| 2250 | acceptedAt: now, |
| 2251 | effects: [], |
| 2252 | events: [ |
| 2253 | { |
| 2254 | id: EventId.make("event:completion-capture-completed"), |
| 2255 | type: "run.updated", |
| 2256 | threadId, |
| 2257 | runId, |
| 2258 | occurredAt: now, |
| 2259 | payload: { |
| 2260 | ...captureRun, |
| 2261 | status: "completed", |
| 2262 | completedAt: now, |
| 2263 | checkpointId, |
| 2264 | }, |
| 2265 | }, |
| 2266 | ], |
| 2267 | }); |
| 2268 | } |
| 2269 | const stored = yield* write(input); |
| 2270 | for (const { event } of stored) { |
| 2271 | if (event.type === "run.updated" && event.payload.id === runId) { |
| 2272 | committedParentRuns.push(event.payload); |
| 2273 | } |
| 2274 | } |
| 2275 | return stored; |
| 2276 | }), |
| 2277 | ), |
| 2278 | ), |
| 2279 | () => orchestrator.recoverDelegatedTasks, |
| 2280 | (spy) => Effect.sync(() => spy.mockRestore()), |
| 2281 | ); |
| 2282 | assert.isTrue(captured); |
| 2283 | assert.equal(committedParentRuns.length, 1); |
| 2284 | const committedParentRun = committedParentRuns[0]; |
| 2285 | assert.equal(committedParentRun?.status, "completed"); |
| 2286 | assert.deepEqual(committedParentRun?.completedAt, now); |
| 2287 | assert.equal(committedParentRun?.checkpointId, checkpointId); |
| 2288 | assert.deepEqual(committedParentRun?.delegatedCompletion?.delivery?.taskIds, [ |
| 2289 | child.taskId, |
| 2290 | ]); |
| 2291 | const finished = yield* orchestrator.getThreadProjection(threadId); |
| 2292 | const completed = finished.runs.find((run) => run.id === runId)!; |
| 2293 | assert.equal(completed.status, "completed"); |
| 2294 | assert.deepEqual(completed.completedAt, now); |
| 2295 | assert.equal(completed.checkpointId, checkpointId); |
| 2296 | assert.deepEqual(completed.delegatedCompletion?.delivery?.taskIds, [child.taskId]); |
| 2297 | const wake = yield* requests.take; |
| 2298 | assert.equal(wake.delegatedCompletion?.parentRunId, runId); |
| 2299 | const delivery = completed.delegatedCompletion!.delivery!; |
| 2300 | yield* orchestrator.dispatch({ |
| 2301 | type: "message.dispatch", |
| 2302 | createdBy: "agent", |
| 2303 | creationSource: "server", |
| 2304 | commandId: CommandId.make("command:completion-capture-delivery"), |
| 2305 | threadId, |
| 2306 | messageId: delivery.messageId, |
| 2307 | text: "Child finished", |
| 2308 | attachments: [], |
| 2309 | delegatedCompletion: { |
| 2310 | parentRunId: runId, |
| 2311 | generation: delivery.generation, |
| 2312 | taskIds: delivery.taskIds, |
| 2313 | }, |
| 2314 | dispatchMode: { type: "queue_after_active" }, |
| 2315 | }); |
| 2316 | const delivered = yield* orchestrator.getThreadProjection(threadId); |
| 2317 | const deliveryRun = delivered.runs.find((run) => run.userMessageId === delivery.messageId); |
| 2318 | assert.isDefined(deliveryRun); |
| 2319 | assert.notEqual(deliveryRun?.status, "queued"); |
| 2320 | }), |
| 2321 | ); |
| 2322 | it.effect("pauses queued follow-up cards and resumes them", () => |
| 2323 | Effect.gen(function* () { |
| 2324 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 2325 | const eventSink = yield* EventSink.EventSinkV2; |
| 2326 | const now = yield* DateTime.now; |
| 2327 | const parentThreadId = ThreadId.make("thread:held-followup-parent"); |
| 2328 | const projectId = ProjectId.make("project:held-followup"); |
| 2329 | const parentRunId = RunId.make("run:held-followup-parent"); |
| 2330 | const rootNodeId = NodeId.make("node:held-followup-parent"); |
| 2331 | yield* seedParentWithTerminalTask({ |
| 2332 | threadId: parentThreadId, |
| 2333 | projectId, |
| 2334 | runId: parentRunId, |
| 2335 | rootNodeId, |
| 2336 | taskId: NodeId.make("node:held-followup-seed"), |
| 2337 | deliveryState: "acknowledged", |
| 2338 | now, |
| 2339 | }); |
| 2340 | const child = yield* seedRestartCancelledChild({ |
| 2341 | parentThreadId, |
| 2342 | projectId, |
| 2343 | parentRunId, |
| 2344 | rootNodeId, |
| 2345 | name: "held-followup-child", |
| 2346 | completionWake: "always", |
| 2347 | continuationPending: false, |
| 2348 | runStatus: "completed", |
| 2349 | now, |
| 2350 | }); |
| 2351 | yield* orchestrator.recoverDelegatedTasks; |
| 2352 | const send = (name: string) => |
| 2353 | orchestrator.dispatch({ |
| 2354 | type: "message.dispatch", |
| 2355 | createdBy: "agent", |
| 2356 | creationSource: "mcp", |
| 2357 | commandId: CommandId.make(`command:held-followup:${name}`), |
| 2358 | threadId: child.childThreadId, |
| 2359 | senderThreadId: parentThreadId, |
| 2360 | messageId: MessageId.make(`message:held-followup:${name}`), |
| 2361 | text: name, |
| 2362 | attachments: [], |
| 2363 | dispatchMode: { type: "queue_after_active" }, |
| 2364 | }); |
| 2365 | yield* send("active"); |
| 2366 | yield* send("queued"); |
| 2367 | const before = yield* orchestrator.getThreadProjection(child.childThreadId); |
| 2368 | const active = before.runs.find((run) => run.ordinal === 2)!; |
| 2369 | const queued = before.runs.find((run) => run.ordinal === 3)!; |
| 2370 | assert.equal(queued.status, "queued"); |
| 2371 | const waitForStatus = (afterSequence: number, status: string) => |
| 2372 | eventSink.stream({ afterSequence, eventType: "subagent.updated" }).pipe( |
| 2373 | Stream.filter( |
| 2374 | (stored) => |
| 2375 | stored.event.type === "subagent.updated" && |
| 2376 | stored.event.payload.id === queued.delegatedTaskId && |
| 2377 | stored.event.payload.status === status, |
| 2378 | ), |
| 2379 | Stream.take(1), |
| 2380 | Stream.runHead, |
| 2381 | ); |
| 2382 | const afterSequence = yield* eventSink.latestSequence(); |
| 2383 | yield* orchestrator.dispatch({ |
| 2384 | type: "thread.stop", |
| 2385 | threadId: child.childThreadId, |
| 2386 | commandId: CommandId.make("command:held-followup:stop"), |
| 2387 | }); |
| 2388 | yield* waitForStatus(afterSequence, "idle"); |
| 2389 | const paused = yield* orchestrator.getThreadProjection(parentThreadId); |
| 2390 | const task = paused.subagents.find((row) => row.id === queued.delegatedTaskId)!; |
| 2391 | assert.equal(task.status, "idle"); |
| 2392 | assert.isNull(task.result); |
| 2393 | assert.equal(paused.nodes.find((row) => row.id === task.id)?.status, "idle"); |
| 2394 | assert.equal( |
| 2395 | paused.turnItems.find((row) => row.type === "subagent" && row.subagentId === task.id) |
| 2396 | ?.status, |
| 2397 | "idle", |
| 2398 | ); |
| 2399 | const endedSequence = yield* eventSink.latestSequence(); |
| 2400 | yield* eventSink.write({ |
| 2401 | events: [ |
| 2402 | { |
| 2403 | id: EventId.make("event:held-followup-active-ended"), |
| 2404 | type: "run.updated", |
| 2405 | threadId: child.childThreadId, |
| 2406 | runId: active.id, |
| 2407 | occurredAt: now, |
| 2408 | payload: { ...active, status: "interrupted", startedAt: now, completedAt: now }, |
| 2409 | }, |
| 2410 | ], |
| 2411 | }); |
| 2412 | const activeTask = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( |
| 2413 | (row) => row.id === active.delegatedTaskId, |
| 2414 | )!; |
| 2415 | if (activeTask.result === null) { |
| 2416 | yield* eventSink |
| 2417 | .stream({ afterSequence: endedSequence, eventType: "subagent.updated" }) |
| 2418 | .pipe( |
| 2419 | Stream.filter( |
| 2420 | (stored) => |
| 2421 | stored.event.type === "subagent.updated" && |
| 2422 | stored.event.payload.id === active.delegatedTaskId && |
| 2423 | stored.event.payload.status === "interrupted", |
| 2424 | ), |
| 2425 | Stream.take(1), |
| 2426 | Stream.runHead, |
| 2427 | ); |
| 2428 | } |
| 2429 | const shell = (yield* orchestrator.getShellSnapshot()).threads.find( |
| 2430 | (row) => row.id === parentThreadId, |
| 2431 | )!; |
| 2432 | assert.isFalse((shell.pendingBackgroundTasks ?? []).some((row) => row.taskId === task.id)); |
| 2433 | const resumeSequence = yield* eventSink.latestSequence(); |
| 2434 | yield* orchestrator.dispatch({ |
| 2435 | type: "queue.resume", |
| 2436 | threadId: child.childThreadId, |
| 2437 | commandId: CommandId.make("command:held-followup:resume"), |
| 2438 | }); |
| 2439 | yield* waitForStatus(resumeSequence, "running"); |
| 2440 | const resumed = yield* orchestrator.getThreadProjection(parentThreadId); |
| 2441 | assert.equal(resumed.subagents.find((row) => row.id === task.id)?.status, "running"); |
| 2442 | assert.isNull(resumed.subagents.find((row) => row.id === task.id)?.result); |
| 2443 | |
| 2444 | const started = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find( |
| 2445 | (run) => run.id === queued.id, |
| 2446 | )!; |
| 2447 | const waitingSequence = yield* eventSink.latestSequence(); |
| 2448 | yield* eventSink.write({ |
| 2449 | events: [ |
| 2450 | { |
| 2451 | id: EventId.make("event:held-followup-waiting"), |
| 2452 | type: "run.updated", |
| 2453 | threadId: child.childThreadId, |
| 2454 | runId: started.id, |
| 2455 | occurredAt: now, |
| 2456 | payload: { ...started, status: "waiting" }, |
| 2457 | }, |
| 2458 | ], |
| 2459 | }); |
| 2460 | yield* waitForStatus(waitingSequence, "waiting"); |
| 2461 | const waiting = yield* orchestrator.getThreadProjection(parentThreadId); |
| 2462 | assert.equal(waiting.subagents.find((row) => row.id === task.id)?.status, "waiting"); |
| 2463 | assert.equal(waiting.nodes.find((row) => row.id === task.id)?.status, "waiting"); |
| 2464 | }), |
| 2465 | ); |
| 2466 | |
| 2467 | it.effect("checkpoint retry releases a pending sibling after its wake offer failed", () => |
| 2468 | Effect.gen(function* () { |
| 2469 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 2470 | const sink = yield* EventSink.EventSinkV2; |
| 2471 | const outbox = yield* EffectOutbox.EffectOutboxV2; |
| 2472 | const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; |
| 2473 | const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; |
| 2474 | const now = yield* DateTime.now; |
| 2475 | const threadId = ThreadId.make("thread:checkpoint-own-delivery"); |
| 2476 | const originRunId = RunId.make("run:checkpoint-own-delivery:origin"); |
| 2477 | const firstId = NodeId.make("node:checkpoint-own-delivery:first"); |
| 2478 | const siblingId = NodeId.make("node:checkpoint-own-delivery:sibling"); |
| 2479 | yield* seedParentWithTerminalTask({ |
| 2480 | threadId, |
| 2481 | projectId: ProjectId.make("project:checkpoint-own-delivery"), |
| 2482 | runId: originRunId, |
| 2483 | rootNodeId: NodeId.make("node:checkpoint-own-delivery:root"), |
| 2484 | taskId: firstId, |
| 2485 | deliveryState: "claimed", |
| 2486 | deliveryTaskIds: [firstId], |
| 2487 | parentRunStatus: "completed", |
| 2488 | now, |
| 2489 | }); |
| 2490 | const original = yield* orchestrator.getThreadProjection(threadId); |
| 2491 | const first = original.subagents.find((task) => task.id === firstId)!; |
| 2492 | const delivery = original.runs.find((run) => run.id === originRunId)!.delegatedCompletion! |
| 2493 | .delivery!; |
| 2494 | yield* orchestrator.dispatch({ |
| 2495 | type: "message.dispatch", |
| 2496 | createdBy: "agent", |
| 2497 | creationSource: "server", |
| 2498 | commandId: reconcileCommandId("checkpoint-own-delivery-wake"), |
| 2499 | threadId, |
| 2500 | messageId: delivery.messageId, |
| 2501 | text: "First result", |
| 2502 | attachments: [], |
| 2503 | dispatchMode: { type: "queue_after_active" }, |
| 2504 | delegatedCompletion: { |
| 2505 | parentRunId: originRunId, |
| 2506 | generation: delivery.generation, |
| 2507 | taskIds: delivery.taskIds, |
| 2508 | }, |
| 2509 | }); |
| 2510 | const started = yield* orchestrator.getThreadProjection(threadId); |
| 2511 | const run = started.runs.find( |
| 2512 | (candidate) => candidate.userMessageId === delivery.messageId, |
| 2513 | )!; |
| 2514 | const scopeId = started.nodes.find( |
| 2515 | (node) => node.id === run.rootNodeId, |
| 2516 | )!.checkpointScopeId!; |
| 2517 | yield* outbox.cancelUnsettled({ |
| 2518 | threadId, |
| 2519 | effectTypes: ["provider-turn.start"], |
| 2520 | reason: "The delivery turn already completed.", |
| 2521 | }); |
| 2522 | const commandId = reconcileCommandId("checkpoint-own-delivery-finished"); |
| 2523 | yield* sink.writeWithEffects({ |
| 2524 | commandId, |
| 2525 | events: [ |
| 2526 | { |
| 2527 | id: EventId.make("event:checkpoint-own-delivery:sibling"), |
| 2528 | type: "subagent.updated", |
| 2529 | threadId, |
| 2530 | runId: originRunId, |
| 2531 | nodeId: siblingId, |
| 2532 | occurredAt: now, |
| 2533 | payload: { |
| 2534 | ...first, |
| 2535 | id: siblingId, |
| 2536 | completionDelivery: { state: "pending", observedByRunId: null }, |
| 2537 | }, |
| 2538 | }, |
| 2539 | { |
| 2540 | id: EventId.make("event:checkpoint-own-delivery:finished"), |
| 2541 | type: "run.updated", |
| 2542 | threadId, |
| 2543 | runId: run.id, |
| 2544 | occurredAt: now, |
| 2545 | payload: { |
| 2546 | ...run, |
| 2547 | status: "completed", |
| 2548 | startedAt: now, |
| 2549 | completedAt: now, |
| 2550 | checkpointId: CheckpointId.make("checkpoint:own-delivery"), |
| 2551 | }, |
| 2552 | }, |
| 2553 | ], |
| 2554 | effects: [ |
| 2555 | { |
| 2556 | id: "effect:checkpoint-own-delivery", |
| 2557 | commandId, |
| 2558 | threadId, |
| 2559 | request: { type: "checkpoint.capture", runId: run.id, scopeId }, |
| 2560 | }, |
| 2561 | ], |
| 2562 | }); |
| 2563 | using offer = vi |
| 2564 | .spyOn(requests, "offer") |
| 2565 | .mockImplementationOnce(() => Effect.die("successor wake offer failed")); |
| 2566 | const firstAttempt = yield* Effect.exit( |
| 2567 | orchestrator.recoverDelegatedTask(threadId, { completedRunId: run.id }), |
| 2568 | ); |
| 2569 | assert.isTrue(Exit.isFailure(firstAttempt)); |
| 2570 | const reserved = (yield* orchestrator.getThreadProjection(threadId)).runs.find( |
| 2571 | (candidate) => candidate.id === originRunId, |
| 2572 | )!.delegatedCompletion!.delivery!; |
| 2573 | assert.deepEqual(reserved.taskIds, [siblingId]); |
| 2574 | yield* worker.drain(); |
| 2575 | assert.lengthOf(offer.mock.calls, 2); |
| 2576 | const completed = yield* orchestrator.getThreadProjection(threadId); |
| 2577 | assert.equal( |
| 2578 | completed.subagents.find((task) => task.id === firstId)?.completionDelivery?.state, |
| 2579 | "delivered", |
| 2580 | ); |
| 2581 | assert.equal( |
| 2582 | completed.subagents.find((task) => task.id === siblingId)?.completionDelivery?.state, |
| 2583 | "claimed", |
| 2584 | ); |
| 2585 | assert.deepEqual( |
| 2586 | completed.runs.find((candidate) => candidate.id === originRunId)?.delegatedCompletion |
| 2587 | ?.delivery?.taskIds, |
| 2588 | [siblingId], |
| 2589 | ); |
| 2590 | let wake = yield* requests.take; |
| 2591 | while (wake.threadId !== threadId) wake = yield* requests.take; |
| 2592 | assert.equal(wake.delegatedCompletion?.generation, delivery.generation + 1); |
| 2593 | }), |
| 2594 | ); |
| 2595 | |
| 2596 | it.effect.each(["live", "checkpoint-retry", "checkpoint-offer-retry"] as const)( |
| 2597 | "delivers the run a background command's wake starts after the original result (%s)", |
| 2598 | (delivery) => |
| 2599 | Effect.gen(function* () { |
| 2600 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 2601 | const eventSink = yield* EventSink.EventSinkV2; |
| 2602 | const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; |
| 2603 | const now = yield* DateTime.now; |
| 2604 | const parentThreadId = ThreadId.make(`thread:background-wake-parent-${delivery}`); |
| 2605 | const projectId = ProjectId.make(`project:background-wake-${delivery}`); |
| 2606 | const parentRunId = RunId.make(`run:background-wake-parent-${delivery}`); |
| 2607 | const rootNodeId = NodeId.make(`node:background-wake-parent-${delivery}`); |
| 2608 | yield* seedParentWithTerminalTask({ |
| 2609 | threadId: parentThreadId, |
| 2610 | projectId, |
| 2611 | runId: parentRunId, |
| 2612 | rootNodeId, |
| 2613 | taskId: NodeId.make(`node:background-wake-seed-${delivery}`), |
| 2614 | deliveryState: "acknowledged", |
| 2615 | now, |
| 2616 | }); |
| 2617 | const child = yield* seedRestartCancelledChild({ |
| 2618 | parentThreadId, |
| 2619 | projectId, |
| 2620 | parentRunId, |
| 2621 | rootNodeId, |
| 2622 | name: `background-wake-child-${delivery}`, |
| 2623 | completionWake: "always", |
| 2624 | continuationPending: false, |
| 2625 | runStatus: "completed", |
| 2626 | now, |
| 2627 | }); |
| 2628 | yield* orchestrator.recoverDelegatedTasks; |
| 2629 | let firstWake = yield* requests.take; |
| 2630 | while (firstWake.threadId !== parentThreadId) firstWake = yield* requests.take; |
| 2631 | yield* orchestrator.dispatch({ |
| 2632 | type: "delegated_task.completion-delivery.acknowledge", |
| 2633 | commandId: CommandId.make(`command:background-wake-ack-${delivery}`), |
| 2634 | parentThreadId, |
| 2635 | taskId: child.taskId, |
| 2636 | observedByRunId: parentRunId, |
| 2637 | }); |
| 2638 | const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.find( |
| 2639 | (run) => run.id === parentRunId, |
| 2640 | )!; |
| 2641 | yield* eventSink.write({ |
| 2642 | commandId: reconcileCommandId(`background-wake-parent-finished-${delivery}`), |
| 2643 | events: [ |
| 2644 | { |
| 2645 | id: EventId.make(`event:background-wake-parent-finished-${delivery}`), |
| 2646 | type: "run.updated", |
| 2647 | threadId: parentThreadId, |
| 2648 | runId: parentRunId, |
| 2649 | occurredAt: now, |
| 2650 | payload: { ...parentRun, status: "completed", completedAt: now }, |
| 2651 | }, |
| 2652 | ], |
| 2653 | }); |
| 2654 | |
| 2655 | // The shape ProviderContinuationService dispatches when the provider |
| 2656 | // reports a background command (wire kind `background_command`) ended. |
| 2657 | const wakeMessageId = MessageId.make(`message:background-wake-${delivery}`); |
| 2658 | yield* orchestrator.dispatch({ |
| 2659 | type: "message.dispatch", |
| 2660 | createdBy: "agent", |
| 2661 | creationSource: "provider", |
| 2662 | commandId: |
| 2663 | delivery === "live" |
| 2664 | ? CommandId.make(`command:background-wake-${delivery}`) |
| 2665 | : reconcileCommandId(`background-wake-${delivery}`), |
| 2666 | threadId: child.childThreadId, |
| 2667 | messageId: wakeMessageId, |
| 2668 | text: "Background activity updated", |
| 2669 | // A monitor that ends is finished work too; only its per-line updates are progress. |
| 2670 | notification: |
| 2671 | delivery === "live" |
| 2672 | ? { |
| 2673 | source: { kind: "monitor" }, |
| 2674 | outcome: "completed", |
| 2675 | summary: 'Monitor "sleep 60" finished', |
| 2676 | } |
| 2677 | : { |
| 2678 | source: { kind: "command" }, |
| 2679 | outcome: "completed", |
| 2680 | summary: 'Command "sleep 60" finished', |
| 2681 | }, |
| 2682 | attachments: [], |
| 2683 | dispatchMode: { type: "queue_after_active" }, |
| 2684 | }); |
| 2685 | const childProjection = yield* orchestrator.getThreadProjection(child.childThreadId); |
| 2686 | const run = childProjection.runs.find((row) => row.userMessageId === wakeMessageId)!; |
| 2687 | const scopeId = childProjection.nodes.find( |
| 2688 | (node) => node.id === run.rootNodeId, |
| 2689 | )!.checkpointScopeId!; |
| 2690 | const commandId = |
| 2691 | delivery === "live" |
| 2692 | ? CommandId.make(`command:background-wake-finished-${delivery}`) |
| 2693 | : reconcileCommandId(`background-wake-finished-${delivery}`); |
| 2694 | if (delivery !== "live") { |
| 2695 | const outbox = yield* EffectOutbox.EffectOutboxV2; |
| 2696 | yield* outbox.cancelUnsettled({ |
| 2697 | threadId: child.childThreadId, |
| 2698 | effectTypes: ["provider-turn.start"], |
| 2699 | reason: "The turn already finished before its checkpoint retry.", |
| 2700 | }); |
| 2701 | } |
| 2702 | const afterSequence = yield* eventSink.latestSequence(); |
| 2703 | assert.isDefined(run.delegatedTaskId); |
| 2704 | yield* eventSink.writeWithEffects({ |
| 2705 | commandId, |
| 2706 | effects: |
| 2707 | delivery === "live" |
| 2708 | ? [] |
| 2709 | : [ |
| 2710 | { |
| 2711 | id: `effect:background-wake-checkpoint-retry-${delivery}`, |
| 2712 | commandId, |
| 2713 | threadId: child.childThreadId, |
| 2714 | request: { type: "checkpoint.capture", runId: run.id, scopeId }, |
| 2715 | }, |
| 2716 | ], |
| 2717 | events: [ |
| 2718 | { |
| 2719 | id: EventId.make(`event:background-wake-result-${delivery}`), |
| 2720 | type: "message.updated", |
| 2721 | threadId: child.childThreadId, |
| 2722 | runId: run.id, |
| 2723 | occurredAt: now, |
| 2724 | payload: { |
| 2725 | id: MessageId.make(`message:background-wake-result-${delivery}`), |
| 2726 | threadId: child.childThreadId, |
| 2727 | runId: run.id, |
| 2728 | nodeId: null, |
| 2729 | role: "assistant", |
| 2730 | text: "The background command finished cleanly.", |
| 2731 | attachments: [], |
| 2732 | streaming: false, |
| 2733 | createdBy: "agent", |
| 2734 | creationSource: "server", |
| 2735 | createdAt: now, |
| 2736 | updatedAt: now, |
| 2737 | }, |
| 2738 | }, |
| 2739 | { |
| 2740 | id: EventId.make(`event:background-wake-finished-${delivery}`), |
| 2741 | type: "run.updated", |
| 2742 | threadId: child.childThreadId, |
| 2743 | runId: run.id, |
| 2744 | occurredAt: now, |
| 2745 | payload: { |
| 2746 | ...run, |
| 2747 | status: "completed", |
| 2748 | startedAt: now, |
| 2749 | completedAt: now, |
| 2750 | checkpointId: |
| 2751 | delivery === "live" |
| 2752 | ? null |
| 2753 | : CheckpointId.make(`checkpoint:background-wake-${delivery}`), |
| 2754 | }, |
| 2755 | }, |
| 2756 | ], |
| 2757 | }); |
| 2758 | if (delivery !== "live") { |
| 2759 | const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; |
| 2760 | if (delivery === "checkpoint-offer-retry") { |
| 2761 | using offer = vi |
| 2762 | .spyOn(requests, "offer") |
| 2763 | .mockImplementationOnce(() => Effect.die("wake offer failed")); |
| 2764 | const first = yield* Effect.exit( |
| 2765 | orchestrator.recoverDelegatedTask(child.childThreadId), |
| 2766 | ); |
| 2767 | assert.isTrue(Exit.isFailure(first)); |
| 2768 | const committed = (yield* orchestrator.getThreadProjection( |
| 2769 | parentThreadId, |
| 2770 | )).subagents.find((task) => task.id === run.delegatedTaskId)!; |
| 2771 | assert.equal(committed.result, "The background command finished cleanly."); |
| 2772 | yield* worker.drain(); |
| 2773 | assert.lengthOf(offer.mock.calls, 2); |
| 2774 | } else { |
| 2775 | yield* worker.drain(); |
| 2776 | } |
| 2777 | } else { |
| 2778 | yield* eventSink.stream({ afterSequence, eventType: "subagent.updated" }).pipe( |
| 2779 | Stream.filter( |
| 2780 | (stored) => |
| 2781 | stored.event.type === "subagent.updated" && |
| 2782 | stored.event.payload.id === run.delegatedTaskId && |
| 2783 | stored.event.payload.status === "completed", |
| 2784 | ), |
| 2785 | Stream.take(1), |
| 2786 | Stream.runDrain, |
| 2787 | ); |
| 2788 | } |
| 2789 | const completed = yield* orchestrator.getThreadProjection(parentThreadId); |
| 2790 | const followup = completed.subagents.find((task) => task.id === run.delegatedTaskId)!; |
| 2791 | assert.lengthOf( |
| 2792 | completed.contextTransfers.filter( |
| 2793 | (transfer) => |
| 2794 | transfer.type === "subagent_result" && |
| 2795 | transfer.sourceThreadId === child.childThreadId, |
| 2796 | ), |
| 2797 | 2, |
| 2798 | ); |
| 2799 | assert.equal(followup.status, "completed"); |
| 2800 | assert.equal(followup.result, "The background command finished cleanly."); |
| 2801 | assert.equal(followup.completionDelivery?.state, "claimed"); |
| 2802 | let wake = yield* requests.take; |
| 2803 | while ( |
| 2804 | wake.threadId !== parentThreadId || |
| 2805 | wake.delegatedCompletion?.messageId === firstWake.delegatedCompletion?.messageId |
| 2806 | ) { |
| 2807 | wake = yield* requests.take; |
| 2808 | } |
| 2809 | assert.equal(wake.delegatedCompletion?.parentRunId, parentRunId); |
| 2810 | }), |
| 2811 | ); |
| 2812 | |
| 2813 | // The child's follow-up run sends `t3_thread_send` to its own thread before it ends. |
| 2814 | const startSelfQueuedFollowup = (name: string) => |
| 2815 | Effect.gen(function* () { |
| 2816 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 2817 | const eventSink = yield* EventSink.EventSinkV2; |
| 2818 | const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; |
| 2819 | const now = yield* DateTime.now; |
| 2820 | const parentThreadId = ThreadId.make(`thread:${name}-parent`); |
| 2821 | const projectId = ProjectId.make(`project:${name}`); |
| 2822 | const parentRunId = RunId.make(`run:${name}-parent`); |
| 2823 | const rootNodeId = NodeId.make(`node:${name}-parent`); |
| 2824 | yield* seedParentWithTerminalTask({ |
| 2825 | threadId: parentThreadId, |
| 2826 | projectId, |
| 2827 | runId: parentRunId, |
| 2828 | rootNodeId, |
| 2829 | taskId: NodeId.make(`node:${name}-seed`), |
| 2830 | deliveryState: "acknowledged", |
| 2831 | now, |
| 2832 | }); |
| 2833 | const child = yield* seedRestartCancelledChild({ |
| 2834 | parentThreadId, |
| 2835 | projectId, |
| 2836 | parentRunId, |
| 2837 | rootNodeId, |
| 2838 | name: `${name}-child`, |
| 2839 | completionWake: "always", |
| 2840 | continuationPending: false, |
| 2841 | runStatus: "completed", |
| 2842 | now, |
| 2843 | }); |
| 2844 | yield* orchestrator.recoverDelegatedTasks; |
| 2845 | let firstWake = yield* requests.take; |
| 2846 | while (firstWake.threadId !== parentThreadId) firstWake = yield* requests.take; |
| 2847 | yield* orchestrator.dispatch({ |
| 2848 | type: "delegated_task.completion-delivery.acknowledge", |
| 2849 | commandId: CommandId.make(`command:${name}-ack`), |
| 2850 | parentThreadId, |
| 2851 | taskId: child.taskId, |
| 2852 | observedByRunId: parentRunId, |
| 2853 | }); |
| 2854 | const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.find( |
| 2855 | (run) => run.id === parentRunId, |
| 2856 | )!; |
| 2857 | yield* eventSink.write({ |
| 2858 | commandId: reconcileCommandId(`${name}-parent-finished`), |
| 2859 | events: [ |
| 2860 | { |
| 2861 | id: EventId.make(`event:${name}-parent-finished`), |
| 2862 | type: "run.updated", |
| 2863 | threadId: parentThreadId, |
| 2864 | runId: parentRunId, |
| 2865 | occurredAt: now, |
| 2866 | payload: { ...parentRun, status: "completed", completedAt: now }, |
| 2867 | }, |
| 2868 | ], |
| 2869 | }); |
| 2870 | const followupMessageId = MessageId.make(`message:${name}`); |
| 2871 | yield* orchestrator.dispatch({ |
| 2872 | type: "message.dispatch", |
| 2873 | createdBy: "agent", |
| 2874 | creationSource: "mcp", |
| 2875 | commandId: CommandId.make(`command:${name}`), |
| 2876 | threadId: child.childThreadId, |
| 2877 | senderThreadId: parentThreadId, |
| 2878 | messageId: followupMessageId, |
| 2879 | text: "Check the same work again.", |
| 2880 | attachments: [], |
| 2881 | dispatchMode: { type: "start_immediately" }, |
| 2882 | }); |
| 2883 | const queuedRun = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find( |
| 2884 | (row) => row.userMessageId === followupMessageId, |
| 2885 | )!; |
| 2886 | const followupTaskId = queuedRun.delegatedTaskId; |
| 2887 | assert.isDefined(followupTaskId); |
| 2888 | const followupRun = { ...queuedRun, status: "running" as const, startedAt: now }; |
| 2889 | yield* eventSink.write({ |
| 2890 | commandId: reconcileCommandId(`${name}-running`), |
| 2891 | events: [ |
| 2892 | { |
| 2893 | id: EventId.make(`event:${name}-running`), |
| 2894 | type: "run.updated", |
| 2895 | threadId: child.childThreadId, |
| 2896 | runId: followupRun.id, |
| 2897 | occurredAt: now, |
| 2898 | payload: followupRun, |
| 2899 | }, |
| 2900 | ], |
| 2901 | }); |
| 2902 | const selfQueuedMessageId = MessageId.make(`message:${name}-self-send`); |
| 2903 | yield* orchestrator.dispatch({ |
| 2904 | type: "message.dispatch", |
| 2905 | createdBy: "agent", |
| 2906 | creationSource: "mcp", |
| 2907 | commandId: CommandId.make(`command:${name}-self-send`), |
| 2908 | threadId: child.childThreadId, |
| 2909 | senderThreadId: child.childThreadId, |
| 2910 | messageId: selfQueuedMessageId, |
| 2911 | text: "Finish the work after this turn.", |
| 2912 | attachments: [], |
| 2913 | dispatchMode: { type: "queue_after_active" }, |
| 2914 | }); |
| 2915 | const selfQueuedRun = (yield* orchestrator.getThreadProjection( |
| 2916 | child.childThreadId, |
| 2917 | )).runs.find((row) => row.userMessageId === selfQueuedMessageId)!; |
| 2918 | assert.equal(selfQueuedRun.status, "queued"); |
| 2919 | const finishRun = (step: string, run: OrchestrationV2Run, text: string) => |
| 2920 | eventSink.write({ |
| 2921 | commandId: reconcileCommandId(`${name}-${step}`), |
| 2922 | events: [ |
| 2923 | { |
| 2924 | id: EventId.make(`event:${name}-${step}:result`), |
| 2925 | type: "message.updated", |
| 2926 | threadId: child.childThreadId, |
| 2927 | runId: run.id, |
| 2928 | occurredAt: now, |
| 2929 | payload: { |
| 2930 | id: MessageId.make(`message:${name}-${step}:result`), |
| 2931 | threadId: child.childThreadId, |
| 2932 | runId: run.id, |
| 2933 | nodeId: null, |
| 2934 | role: "assistant", |
| 2935 | text, |
| 2936 | attachments: [], |
| 2937 | streaming: false, |
| 2938 | createdBy: "agent", |
| 2939 | creationSource: "server", |
| 2940 | createdAt: now, |
| 2941 | updatedAt: now, |
| 2942 | }, |
| 2943 | }, |
| 2944 | { |
| 2945 | id: EventId.make(`event:${name}-${step}:finished`), |
| 2946 | type: "run.updated", |
| 2947 | threadId: child.childThreadId, |
| 2948 | runId: run.id, |
| 2949 | occurredAt: now, |
| 2950 | payload: { ...run, status: "completed", startedAt: now, completedAt: now }, |
| 2951 | }, |
| 2952 | ], |
| 2953 | }); |
| 2954 | const removeSelfQueuedRun = orchestrator.dispatch({ |
| 2955 | type: "queued-run.cancel", |
| 2956 | commandId: CommandId.make(`command:${name}-remove-self-send`), |
| 2957 | threadId: child.childThreadId, |
| 2958 | runId: selfQueuedRun.id, |
| 2959 | }); |
| 2960 | const readFollowup = Effect.gen(function* () { |
| 2961 | yield* orchestrator.recoverDelegatedTasks; |
| 2962 | return (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( |
| 2963 | (task) => task.id === followupTaskId, |
| 2964 | )!; |
| 2965 | }); |
| 2966 | return { |
| 2967 | parentThreadId, |
| 2968 | parentRunId, |
| 2969 | childThreadId: child.childThreadId, |
| 2970 | firstWake, |
| 2971 | followupRun, |
| 2972 | selfQueuedRun, |
| 2973 | finishRun, |
| 2974 | removeSelfQueuedRun, |
| 2975 | readFollowup, |
| 2976 | }; |
| 2977 | }); |
| 2978 | |
| 2979 | it.effect("holds a follow-up open while the child's own subagent still runs", () => |
| 2980 | Effect.gen(function* () { |
| 2981 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 2982 | const eventSink = yield* EventSink.EventSinkV2; |
| 2983 | const now = yield* DateTime.now; |
| 2984 | const parentThreadId = ThreadId.make("thread:followup-grandchild-parent"); |
| 2985 | const projectId = ProjectId.make("project:followup-grandchild"); |
| 2986 | const parentRunId = RunId.make("run:followup-grandchild-parent"); |
| 2987 | const rootNodeId = NodeId.make("node:followup-grandchild-parent"); |
| 2988 | yield* seedParentWithTerminalTask({ |
| 2989 | threadId: parentThreadId, |
| 2990 | projectId, |
| 2991 | runId: parentRunId, |
| 2992 | rootNodeId, |
| 2993 | taskId: NodeId.make("node:followup-grandchild-seed"), |
| 2994 | deliveryState: "acknowledged", |
| 2995 | now, |
| 2996 | }); |
| 2997 | const child = yield* seedRestartCancelledChild({ |
| 2998 | parentThreadId, |
| 2999 | projectId, |
| 3000 | parentRunId, |
| 3001 | rootNodeId, |
| 3002 | name: "followup-grandchild-child", |
| 3003 | completionWake: "always", |
| 3004 | continuationPending: false, |
| 3005 | runStatus: "completed", |
| 3006 | now, |
| 3007 | }); |
| 3008 | yield* orchestrator.recoverDelegatedTasks; |
| 3009 | assert.equal( |
| 3010 | (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( |
| 3011 | (task) => task.id === child.taskId, |
| 3012 | )?.status, |
| 3013 | "completed", |
| 3014 | ); |
| 3015 | |
| 3016 | const followupMessageId = MessageId.make("message:followup-grandchild-send"); |
| 3017 | yield* orchestrator.dispatch({ |
| 3018 | type: "message.dispatch", |
| 3019 | createdBy: "agent", |
| 3020 | creationSource: "mcp", |
| 3021 | commandId: CommandId.make("command:followup-grandchild-send"), |
| 3022 | threadId: child.childThreadId, |
| 3023 | senderThreadId: parentThreadId, |
| 3024 | messageId: followupMessageId, |
| 3025 | text: "Now delegate the next step.", |
| 3026 | attachments: [], |
| 3027 | dispatchMode: { type: "queue_after_active" }, |
| 3028 | }); |
| 3029 | const followupRun = (yield* orchestrator.getThreadProjection( |
| 3030 | child.childThreadId, |
| 3031 | )).runs.find((run) => run.userMessageId === followupMessageId)!; |
| 3032 | assert.isDefined(followupRun.delegatedTaskId); |
| 3033 | const followupTaskId = followupRun.delegatedTaskId!; |
| 3034 | |
| 3035 | const grandchildId = NodeId.make("node:followup-grandchild-grandchild"); |
| 3036 | const grandchild = { |
| 3037 | id: grandchildId, |
| 3038 | threadId: child.childThreadId, |
| 3039 | runId: followupRun.id, |
| 3040 | parentNodeId: NodeId.make("node:followup-grandchild-child-root"), |
| 3041 | origin: "app_owned" as const, |
| 3042 | createdBy: "agent" as const, |
| 3043 | driver, |
| 3044 | providerInstanceId: modelSelection.instanceId, |
| 3045 | providerThreadId: null, |
| 3046 | childThreadId: ThreadId.make("thread:followup-grandchild-grandchild"), |
| 3047 | nativeTaskRef: null, |
| 3048 | prompt: "Do the next step.", |
| 3049 | title: null, |
| 3050 | model: null, |
| 3051 | completionWake: "always" as const, |
| 3052 | status: "running" as const, |
| 3053 | result: null, |
| 3054 | startedAt: now, |
| 3055 | completedAt: null, |
| 3056 | updatedAt: now, |
| 3057 | }; |
| 3058 | yield* eventSink.write({ |
| 3059 | commandId: reconcileCommandId("followup-grandchild-turn-ended"), |
| 3060 | events: [ |
| 3061 | { |
| 3062 | id: EventId.make("event:followup-grandchild-started"), |
| 3063 | type: "subagent.updated", |
| 3064 | threadId: child.childThreadId, |
| 3065 | runId: followupRun.id, |
| 3066 | nodeId: grandchildId, |
| 3067 | driver, |
| 3068 | providerInstanceId: modelSelection.instanceId, |
| 3069 | occurredAt: now, |
| 3070 | payload: grandchild, |
| 3071 | }, |
| 3072 | { |
| 3073 | id: EventId.make("event:followup-grandchild-run-completed"), |
| 3074 | type: "run.updated", |
| 3075 | threadId: child.childThreadId, |
| 3076 | runId: followupRun.id, |
| 3077 | occurredAt: now, |
| 3078 | payload: { ...followupRun, status: "completed", startedAt: now, completedAt: now }, |
| 3079 | }, |
| 3080 | ], |
| 3081 | }); |
| 3082 | yield* orchestrator.recoverDelegatedTasks; |
| 3083 | |
| 3084 | const waiting = yield* orchestrator.getThreadProjection(parentThreadId); |
| 3085 | const heldFollowup = waiting.subagents.find((task) => task.id === followupTaskId); |
| 3086 | assert.isNull( |
| 3087 | heldFollowup?.result ?? null, |
| 3088 | "the follow-up closed while the child's own subagent was still running", |
| 3089 | ); |
| 3090 | assert.equal(heldFollowup?.status, "running"); |
| 3091 | assert.equal( |
| 3092 | waiting.contextTransfers.filter( |
| 3093 | (transfer) => |
| 3094 | transfer.type === "subagent_result" && |
| 3095 | transfer.sourceThreadId === child.childThreadId, |
| 3096 | ).length, |
| 3097 | 1, |
| 3098 | ); |
| 3099 | |
| 3100 | // Another turn in the child does not release the hold while the grandchild runs. |
| 3101 | for (const status of ["running", "completed"] as const) { |
| 3102 | yield* eventSink.write({ |
| 3103 | commandId: reconcileCommandId(`followup-grandchild-user-${status}`), |
| 3104 | events: [ |
| 3105 | runEvent({ |
| 3106 | threadId: child.childThreadId, |
| 3107 | runId: RunId.make("run:followup-grandchild-user"), |
| 3108 | ordinal: followupRun.ordinal + 1, |
| 3109 | status, |
| 3110 | now, |
| 3111 | }), |
| 3112 | ], |
| 3113 | }); |
| 3114 | yield* orchestrator.recoverDelegatedTasks; |
| 3115 | assert.isNull( |
| 3116 | (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( |
| 3117 | (task) => task.id === followupTaskId, |
| 3118 | )?.result ?? null, |
| 3119 | `the follow-up closed while another child run was ${status}`, |
| 3120 | ); |
| 3121 | } |
| 3122 | |
| 3123 | // The grandchild's completion wakes the child in a run without a delegatedTaskId. |
| 3124 | const wakeRunId = RunId.make("run:followup-grandchild-wake"); |
| 3125 | const wakeRun = (status: "running" | "completed") => |
| 3126 | runEvent({ |
| 3127 | threadId: child.childThreadId, |
| 3128 | runId: wakeRunId, |
| 3129 | ordinal: followupRun.ordinal + 2, |
| 3130 | status, |
| 3131 | now, |
| 3132 | }); |
| 3133 | yield* eventSink.write({ |
| 3134 | commandId: reconcileCommandId("followup-grandchild-settled"), |
| 3135 | events: [ |
| 3136 | { |
| 3137 | id: EventId.make("event:followup-grandchild-completed"), |
| 3138 | type: "subagent.updated", |
| 3139 | threadId: child.childThreadId, |
| 3140 | runId: followupRun.id, |
| 3141 | nodeId: grandchildId, |
| 3142 | driver, |
| 3143 | providerInstanceId: modelSelection.instanceId, |
| 3144 | occurredAt: now, |
| 3145 | payload: { |
| 3146 | ...grandchild, |
| 3147 | status: "completed", |
| 3148 | result: "grandchild finished", |
| 3149 | completionDelivery: { state: "acknowledged", observedByRunId: wakeRunId }, |
| 3150 | completedAt: now, |
| 3151 | }, |
| 3152 | }, |
| 3153 | { |
| 3154 | id: EventId.make("event:followup-grandchild-wake-message"), |
| 3155 | type: "message.updated", |
| 3156 | threadId: child.childThreadId, |
| 3157 | runId: wakeRunId, |
| 3158 | occurredAt: now, |
| 3159 | payload: { |
| 3160 | id: MessageId.make(`message:${wakeRunId}`), |
| 3161 | threadId: child.childThreadId, |
| 3162 | runId: wakeRunId, |
| 3163 | nodeId: null, |
| 3164 | role: "user", |
| 3165 | text: "Delegated task finished.", |
| 3166 | attachments: [], |
| 3167 | streaming: false, |
| 3168 | createdBy: "agent", |
| 3169 | creationSource: "server", |
| 3170 | delegatedCompletion: { |
| 3171 | parentRunId: followupRun.id, |
| 3172 | generation: 1, |
| 3173 | taskIds: [grandchildId], |
| 3174 | }, |
| 3175 | createdAt: now, |
| 3176 | updatedAt: now, |
| 3177 | }, |
| 3178 | }, |
| 3179 | wakeRun("running"), |
| 3180 | ], |
| 3181 | }); |
| 3182 | yield* orchestrator.recoverDelegatedTasks; |
| 3183 | const waking = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( |
| 3184 | (task) => task.id === followupTaskId, |
| 3185 | ); |
| 3186 | assert.isNull(waking?.result ?? null, "the follow-up closed before the child's wake ended"); |
| 3187 | |
| 3188 | yield* eventSink.write({ |
| 3189 | commandId: reconcileCommandId("followup-grandchild-wake-finished"), |
| 3190 | events: [ |
| 3191 | { |
| 3192 | id: EventId.make("event:followup-grandchild-wake-result"), |
| 3193 | type: "message.updated", |
| 3194 | threadId: child.childThreadId, |
| 3195 | runId: wakeRunId, |
| 3196 | occurredAt: now, |
| 3197 | payload: { |
| 3198 | id: MessageId.make("message:followup-grandchild-wake-result"), |
| 3199 | threadId: child.childThreadId, |
| 3200 | runId: wakeRunId, |
| 3201 | nodeId: null, |
| 3202 | role: "assistant", |
| 3203 | text: "The grandchild finished the next step.", |
| 3204 | attachments: [], |
| 3205 | streaming: false, |
| 3206 | createdBy: "agent", |
| 3207 | creationSource: "server", |
| 3208 | createdAt: now, |
| 3209 | updatedAt: now, |
| 3210 | }, |
| 3211 | }, |
| 3212 | wakeRun("completed"), |
| 3213 | ], |
| 3214 | }); |
| 3215 | yield* orchestrator.recoverDelegatedTasks; |
| 3216 | const settled = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( |
| 3217 | (task) => task.id === followupTaskId, |
| 3218 | ); |
| 3219 | assert.equal(settled?.status, "completed"); |
| 3220 | assert.equal(settled?.result, "The grandchild finished the next step."); |
| 3221 | }), |
| 3222 | ); |
| 3223 | |
| 3224 | it.effect.each(["woken", "cancelled"] as const)( |
| 3225 | "holds a follow-up open while the child's awaited background command runs (%s)", |
| 3226 | (ending) => |
| 3227 | Effect.gen(function* () { |
| 3228 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 3229 | const eventSink = yield* EventSink.EventSinkV2; |
| 3230 | const now = yield* DateTime.now; |
| 3231 | const parentThreadId = ThreadId.make(`thread:followup-command-parent-${ending}`); |
| 3232 | const projectId = ProjectId.make(`project:followup-command-${ending}`); |
| 3233 | const parentRunId = RunId.make(`run:followup-command-parent-${ending}`); |
| 3234 | const rootNodeId = NodeId.make(`node:followup-command-parent-${ending}`); |
| 3235 | yield* seedParentWithTerminalTask({ |
| 3236 | threadId: parentThreadId, |
| 3237 | projectId, |
| 3238 | runId: parentRunId, |
| 3239 | rootNodeId, |
| 3240 | taskId: NodeId.make(`node:followup-command-seed-${ending}`), |
| 3241 | deliveryState: "acknowledged", |
| 3242 | now, |
| 3243 | }); |
| 3244 | const child = yield* seedRestartCancelledChild({ |
| 3245 | parentThreadId, |
| 3246 | projectId, |
| 3247 | parentRunId, |
| 3248 | rootNodeId, |
| 3249 | name: `followup-command-child-${ending}`, |
| 3250 | completionWake: "always", |
| 3251 | continuationPending: false, |
| 3252 | runStatus: "completed", |
| 3253 | now, |
| 3254 | }); |
| 3255 | yield* orchestrator.recoverDelegatedTasks; |
| 3256 | |
| 3257 | const followupMessageId = MessageId.make(`message:followup-command-send-${ending}`); |
| 3258 | yield* orchestrator.dispatch({ |
| 3259 | type: "message.dispatch", |
| 3260 | createdBy: "agent", |
| 3261 | creationSource: "mcp", |
| 3262 | commandId: CommandId.make(`command:followup-command-send-${ending}`), |
| 3263 | threadId: child.childThreadId, |
| 3264 | senderThreadId: parentThreadId, |
| 3265 | messageId: followupMessageId, |
| 3266 | text: "Start the check in the background, wait on it, and reply when it ends.", |
| 3267 | attachments: [], |
| 3268 | dispatchMode: { type: "queue_after_active" }, |
| 3269 | }); |
| 3270 | const followupRun = (yield* orchestrator.getThreadProjection( |
| 3271 | child.childThreadId, |
| 3272 | )).runs.find((run) => run.userMessageId === followupMessageId)!; |
| 3273 | const followupTaskId = followupRun.delegatedTaskId!; |
| 3274 | const parent = yield* orchestrator.getThreadProjection(parentThreadId); |
| 3275 | |
| 3276 | // The child awaits a background command, replies with an interim note and ends its turn. |
| 3277 | yield* eventSink.write({ |
| 3278 | commandId: reconcileCommandId(`followup-command-turn-ended-${ending}`), |
| 3279 | events: [ |
| 3280 | { |
| 3281 | id: EventId.make(`event:followup-command-awaited-${ending}`), |
| 3282 | type: "provider-thread.updated", |
| 3283 | threadId: child.childThreadId, |
| 3284 | runId: followupRun.id, |
| 3285 | driver, |
| 3286 | providerInstanceId: modelSelection.instanceId, |
| 3287 | occurredAt: now, |
| 3288 | payload: { |
| 3289 | ...parent.providerThreads[0]!, |
| 3290 | id: ProviderThreadId.make(`provider-thread:followup-command-child-${ending}`), |
| 3291 | appThreadId: child.childThreadId, |
| 3292 | pendingBackgroundTasks: [ |
| 3293 | { taskId: "sleep-90", kind: "command" as const, awaited: true as const }, |
| 3294 | ], |
| 3295 | }, |
| 3296 | }, |
| 3297 | { |
| 3298 | id: EventId.make(`event:followup-command-interim-${ending}`), |
| 3299 | type: "message.updated", |
| 3300 | threadId: child.childThreadId, |
| 3301 | runId: followupRun.id, |
| 3302 | occurredAt: now, |
| 3303 | payload: { |
| 3304 | id: MessageId.make(`message:followup-command-interim-${ending}`), |
| 3305 | threadId: child.childThreadId, |
| 3306 | runId: followupRun.id, |
| 3307 | nodeId: null, |
| 3308 | role: "assistant", |
| 3309 | text: "INTERIM", |
| 3310 | attachments: [], |
| 3311 | streaming: false, |
| 3312 | createdBy: "agent", |
| 3313 | creationSource: "server", |
| 3314 | createdAt: now, |
| 3315 | updatedAt: now, |
| 3316 | }, |
| 3317 | }, |
| 3318 | { |
| 3319 | id: EventId.make(`event:followup-command-run-completed-${ending}`), |
| 3320 | type: "run.updated", |
| 3321 | threadId: child.childThreadId, |
| 3322 | runId: followupRun.id, |
| 3323 | occurredAt: now, |
| 3324 | payload: { ...followupRun, status: "completed", startedAt: now, completedAt: now }, |
| 3325 | }, |
| 3326 | ], |
| 3327 | }); |
| 3328 | yield* orchestrator.recoverDelegatedTasks; |
| 3329 | |
| 3330 | const held = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( |
| 3331 | (task) => task.id === followupTaskId, |
| 3332 | ); |
| 3333 | assert.isNull( |
| 3334 | held?.result ?? null, |
| 3335 | "the follow-up closed with its interim reply while the awaited command still ran", |
| 3336 | ); |
| 3337 | assert.equal(held?.status, "running"); |
| 3338 | |
| 3339 | if (ending === "cancelled") { |
| 3340 | yield* orchestrator.cancelDelegatedTask({ |
| 3341 | parentThreadId, |
| 3342 | taskId: followupTaskId, |
| 3343 | command: { |
| 3344 | type: "thread.stop", |
| 3345 | commandId: CommandId.make(`command:followup-command-cancel-${ending}`), |
| 3346 | threadId: child.childThreadId, |
| 3347 | stopDelegatedTasks: false, |
| 3348 | }, |
| 3349 | stopDescendants: () => Effect.void, |
| 3350 | }); |
| 3351 | yield* eventSink.write({ |
| 3352 | commandId: reconcileCommandId(`followup-command-stopped-${ending}`), |
| 3353 | events: [ |
| 3354 | { |
| 3355 | id: EventId.make(`event:followup-command-stopped-${ending}`), |
| 3356 | type: "provider-thread.updated", |
| 3357 | threadId: child.childThreadId, |
| 3358 | driver, |
| 3359 | providerInstanceId: modelSelection.instanceId, |
| 3360 | occurredAt: now, |
| 3361 | payload: { |
| 3362 | ...parent.providerThreads[0]!, |
| 3363 | id: ProviderThreadId.make(`provider-thread:followup-command-child-${ending}`), |
| 3364 | appThreadId: child.childThreadId, |
| 3365 | pendingBackgroundTasks: [], |
| 3366 | }, |
| 3367 | }, |
| 3368 | ], |
| 3369 | }); |
| 3370 | yield* orchestrator.recoverDelegatedTasks; |
| 3371 | const stopped = (yield* orchestrator.getThreadProjection( |
| 3372 | parentThreadId, |
| 3373 | )).subagents.find((task) => task.id === followupTaskId); |
| 3374 | assert.equal(stopped?.cancelledRunId, followupRun.id); |
| 3375 | assert.equal(stopped?.status, "interrupted"); |
| 3376 | return; |
| 3377 | } |
| 3378 | |
| 3379 | // The finished command wakes the child; that turn continues the same follow-up. |
| 3380 | const wakeMessageId = MessageId.make(`message:followup-command-wake-${ending}`); |
| 3381 | yield* orchestrator.dispatch({ |
| 3382 | type: "message.dispatch", |
| 3383 | createdBy: "agent", |
| 3384 | creationSource: "server", |
| 3385 | commandId: CommandId.make(`command:followup-command-wake-${ending}`), |
| 3386 | threadId: child.childThreadId, |
| 3387 | messageId: wakeMessageId, |
| 3388 | text: "Background activity updated", |
| 3389 | notification: { |
| 3390 | source: { kind: "command" }, |
| 3391 | outcome: "completed", |
| 3392 | summary: 'Command "sleep 90" finished', |
| 3393 | }, |
| 3394 | attachments: [], |
| 3395 | dispatchMode: { type: "queue_after_active" }, |
| 3396 | }); |
| 3397 | const wakeRun = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find( |
| 3398 | (run) => run.userMessageId === wakeMessageId, |
| 3399 | )!; |
| 3400 | assert.equal(wakeRun.delegatedTaskId, followupTaskId); |
| 3401 | yield* eventSink.write({ |
| 3402 | commandId: reconcileCommandId(`followup-command-wake-ended-${ending}`), |
| 3403 | events: [ |
| 3404 | { |
| 3405 | id: EventId.make(`event:followup-command-settled-${ending}`), |
| 3406 | type: "provider-thread.updated", |
| 3407 | threadId: child.childThreadId, |
| 3408 | driver, |
| 3409 | providerInstanceId: modelSelection.instanceId, |
| 3410 | occurredAt: now, |
| 3411 | payload: { |
| 3412 | ...parent.providerThreads[0]!, |
| 3413 | id: ProviderThreadId.make(`provider-thread:followup-command-child-${ending}`), |
| 3414 | appThreadId: child.childThreadId, |
| 3415 | pendingBackgroundTasks: [], |
| 3416 | }, |
| 3417 | }, |
| 3418 | { |
| 3419 | id: EventId.make(`event:followup-command-final-${ending}`), |
| 3420 | type: "message.updated", |
| 3421 | threadId: child.childThreadId, |
| 3422 | runId: wakeRun.id, |
| 3423 | occurredAt: now, |
| 3424 | payload: { |
| 3425 | id: MessageId.make(`message:followup-command-final-${ending}`), |
| 3426 | threadId: child.childThreadId, |
| 3427 | runId: wakeRun.id, |
| 3428 | nodeId: null, |
| 3429 | role: "assistant", |
| 3430 | text: "FINAL CHECK-OK", |
| 3431 | attachments: [], |
| 3432 | streaming: false, |
| 3433 | createdBy: "agent", |
| 3434 | creationSource: "server", |
| 3435 | createdAt: now, |
| 3436 | updatedAt: now, |
| 3437 | }, |
| 3438 | }, |
| 3439 | { |
| 3440 | id: EventId.make(`event:followup-command-wake-completed-${ending}`), |
| 3441 | type: "run.updated", |
| 3442 | threadId: child.childThreadId, |
| 3443 | runId: wakeRun.id, |
| 3444 | occurredAt: now, |
| 3445 | payload: { ...wakeRun, status: "completed", startedAt: now, completedAt: now }, |
| 3446 | }, |
| 3447 | ], |
| 3448 | }); |
| 3449 | yield* orchestrator.recoverDelegatedTasks; |
| 3450 | |
| 3451 | const finished = yield* orchestrator.getThreadProjection(parentThreadId); |
| 3452 | const childTasks = finished.subagents.filter( |
| 3453 | (task) => task.childThreadId === child.childThreadId, |
| 3454 | ); |
| 3455 | assert.deepEqual( |
| 3456 | childTasks.map((task) => task.id).toSorted(), |
| 3457 | [child.taskId, followupTaskId].toSorted(), |
| 3458 | ); |
| 3459 | const followup = childTasks.find((task) => task.id === followupTaskId); |
| 3460 | assert.equal(followup?.status, "completed"); |
| 3461 | assert.equal(followup?.result, "FINAL CHECK-OK"); |
| 3462 | }), |
| 3463 | ); |
| 3464 | |
| 3465 | it.effect("settles a follow-up whose own run started no background work", () => |
| 3466 | Effect.gen(function* () { |
| 3467 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 3468 | const eventSink = yield* EventSink.EventSinkV2; |
| 3469 | const now = yield* DateTime.now; |
| 3470 | const parentThreadId = ThreadId.make("thread:followup-earlier-work-parent"); |
| 3471 | const projectId = ProjectId.make("project:followup-earlier-work"); |
| 3472 | const parentRunId = RunId.make("run:followup-earlier-work-parent"); |
| 3473 | const rootNodeId = NodeId.make("node:followup-earlier-work-parent"); |
| 3474 | yield* seedParentWithTerminalTask({ |
| 3475 | threadId: parentThreadId, |
| 3476 | projectId, |
| 3477 | runId: parentRunId, |
| 3478 | rootNodeId, |
| 3479 | taskId: NodeId.make("node:followup-earlier-work-seed"), |
| 3480 | deliveryState: "acknowledged", |
| 3481 | now, |
| 3482 | }); |
| 3483 | const child = yield* seedRestartCancelledChild({ |
| 3484 | parentThreadId, |
| 3485 | projectId, |
| 3486 | parentRunId, |
| 3487 | rootNodeId, |
| 3488 | name: "followup-earlier-work-child", |
| 3489 | completionWake: "always", |
| 3490 | continuationPending: false, |
| 3491 | runStatus: "completed", |
| 3492 | now, |
| 3493 | }); |
| 3494 | yield* orchestrator.recoverDelegatedTasks; |
| 3495 | const parent = yield* orchestrator.getThreadProjection(parentThreadId); |
| 3496 | const childProviderThread = { |
| 3497 | ...parent.providerThreads[0]!, |
| 3498 | id: ProviderThreadId.make("provider-thread:followup-earlier-work-child"), |
| 3499 | appThreadId: child.childThreadId, |
| 3500 | }; |
| 3501 | |
| 3502 | // An earlier run of the child left an awaited build running. |
| 3503 | yield* eventSink.write({ |
| 3504 | commandId: reconcileCommandId("followup-earlier-work-build"), |
| 3505 | events: [ |
| 3506 | { |
| 3507 | id: EventId.make("event:followup-earlier-work-build"), |
| 3508 | type: "provider-thread.updated", |
| 3509 | threadId: child.childThreadId, |
| 3510 | runId: child.childRunId, |
| 3511 | driver, |
| 3512 | providerInstanceId: modelSelection.instanceId, |
| 3513 | occurredAt: now, |
| 3514 | payload: { |
| 3515 | ...childProviderThread, |
| 3516 | pendingBackgroundTasks: [ |
| 3517 | { taskId: "build", kind: "command" as const, awaited: true as const }, |
| 3518 | ], |
| 3519 | }, |
| 3520 | }, |
| 3521 | ], |
| 3522 | }); |
| 3523 | |
| 3524 | const followupMessageId = MessageId.make("message:followup-earlier-work-send"); |
| 3525 | yield* orchestrator.dispatch({ |
| 3526 | type: "message.dispatch", |
| 3527 | createdBy: "agent", |
| 3528 | creationSource: "mcp", |
| 3529 | commandId: CommandId.make("command:followup-earlier-work-send"), |
| 3530 | threadId: child.childThreadId, |
| 3531 | senderThreadId: parentThreadId, |
| 3532 | messageId: followupMessageId, |
| 3533 | text: "Reply with exactly PONG.", |
| 3534 | attachments: [], |
| 3535 | dispatchMode: { type: "queue_after_active" }, |
| 3536 | }); |
| 3537 | const followupRun = (yield* orchestrator.getThreadProjection( |
| 3538 | child.childThreadId, |
| 3539 | )).runs.find((run) => run.userMessageId === followupMessageId)!; |
| 3540 | const followupTaskId = followupRun.delegatedTaskId!; |
| 3541 | |
| 3542 | // The follow-up's run answers and ends without starting any background work. |
| 3543 | yield* eventSink.write({ |
| 3544 | commandId: reconcileCommandId("followup-earlier-work-answered"), |
| 3545 | events: [ |
| 3546 | // The provider resends its whole roster during the follow-up's run. |
| 3547 | { |
| 3548 | id: EventId.make("event:followup-earlier-work-roster"), |
| 3549 | type: "provider-thread.updated", |
| 3550 | threadId: child.childThreadId, |
| 3551 | runId: followupRun.id, |
| 3552 | driver, |
| 3553 | providerInstanceId: modelSelection.instanceId, |
| 3554 | occurredAt: now, |
| 3555 | payload: { |
| 3556 | ...childProviderThread, |
| 3557 | pendingBackgroundTasks: [ |
| 3558 | { taskId: "build", kind: "command" as const, awaited: true as const }, |
| 3559 | ], |
| 3560 | }, |
| 3561 | }, |
| 3562 | { |
| 3563 | id: EventId.make("event:followup-earlier-work-answer"), |
| 3564 | type: "message.updated", |
| 3565 | threadId: child.childThreadId, |
| 3566 | runId: followupRun.id, |
| 3567 | occurredAt: now, |
| 3568 | payload: { |
| 3569 | id: MessageId.make("message:followup-earlier-work-answer"), |
| 3570 | threadId: child.childThreadId, |
| 3571 | runId: followupRun.id, |
| 3572 | nodeId: null, |
| 3573 | role: "assistant", |
| 3574 | text: "PONG", |
| 3575 | attachments: [], |
| 3576 | streaming: false, |
| 3577 | createdBy: "agent", |
| 3578 | creationSource: "server", |
| 3579 | createdAt: now, |
| 3580 | updatedAt: now, |
| 3581 | }, |
| 3582 | }, |
| 3583 | { |
| 3584 | id: EventId.make("event:followup-earlier-work-run-completed"), |
| 3585 | type: "run.updated", |
| 3586 | threadId: child.childThreadId, |
| 3587 | runId: followupRun.id, |
| 3588 | occurredAt: now, |
| 3589 | payload: { ...followupRun, status: "completed", startedAt: now, completedAt: now }, |
| 3590 | }, |
| 3591 | ], |
| 3592 | }); |
| 3593 | yield* orchestrator.recoverDelegatedTasks; |
| 3594 | |
| 3595 | const followup = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( |
| 3596 | (task) => task.id === followupTaskId, |
| 3597 | ); |
| 3598 | assert.equal(followup?.status, "completed"); |
| 3599 | assert.equal(followup?.result, "PONG"); |
| 3600 | }), |
| 3601 | ); |
| 3602 | |
| 3603 | it.effect("tracks a user's follow-up but not the compaction sent ahead of it", () => |
| 3604 | Effect.gen(function* () { |
| 3605 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 3606 | const now = yield* DateTime.now; |
| 3607 | const parentThreadId = ThreadId.make("thread:followup-compact-parent"); |
| 3608 | const projectId = ProjectId.make("project:followup-compact"); |
| 3609 | const parentRunId = RunId.make("run:followup-compact-parent"); |
| 3610 | const rootNodeId = NodeId.make("node:followup-compact-parent"); |
| 3611 | yield* seedParentWithTerminalTask({ |
| 3612 | threadId: parentThreadId, |
| 3613 | projectId, |
| 3614 | runId: parentRunId, |
| 3615 | rootNodeId, |
| 3616 | taskId: NodeId.make("node:followup-compact-seed"), |
| 3617 | deliveryState: "acknowledged", |
| 3618 | now, |
| 3619 | }); |
| 3620 | const child = yield* seedRestartCancelledChild({ |
| 3621 | parentThreadId, |
| 3622 | projectId, |
| 3623 | parentRunId, |
| 3624 | rootNodeId, |
| 3625 | name: "followup-compact-child", |
| 3626 | completionWake: "always", |
| 3627 | continuationPending: false, |
| 3628 | runStatus: "completed", |
| 3629 | now, |
| 3630 | }); |
| 3631 | yield* orchestrator.recoverDelegatedTasks; |
| 3632 | const tasksBefore = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents |
| 3633 | .length; |
| 3634 | |
| 3635 | const send = (name: string, text: string) => |
| 3636 | orchestrator.dispatch({ |
| 3637 | type: "message.dispatch", |
| 3638 | createdBy: "user", |
| 3639 | creationSource: "web", |
| 3640 | commandId: CommandId.make(`command:followup-compact-${name}`), |
| 3641 | threadId: child.childThreadId, |
| 3642 | messageId: MessageId.make(`message:followup-compact-${name}`), |
| 3643 | text, |
| 3644 | attachments: [], |
| 3645 | dispatchMode: { type: "queue_after_active" }, |
| 3646 | }); |
| 3647 | yield* send("compact", "/compact"); |
| 3648 | yield* send("ask", "Write the next packet."); |
| 3649 | |
| 3650 | const runs = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs; |
| 3651 | const runOf = (name: string) => |
| 3652 | runs.find((run) => run.userMessageId === `message:followup-compact-${name}`); |
| 3653 | assert.isDefined(runOf("compact")); |
| 3654 | assert.isUndefined(runOf("compact")?.delegatedTaskId); |
| 3655 | assert.isDefined(runOf("ask")?.delegatedTaskId); |
| 3656 | assert.equal( |
| 3657 | (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.length, |
| 3658 | tasksBefore + 1, |
| 3659 | ); |
| 3660 | }), |
| 3661 | ); |
| 3662 | |
| 3663 | it.effect("settles a follow-up when its own work ends while another's still runs", () => |
| 3664 | Effect.gen(function* () { |
| 3665 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 3666 | const eventSink = yield* EventSink.EventSinkV2; |
| 3667 | const now = yield* DateTime.now; |
| 3668 | const parentThreadId = ThreadId.make("thread:followup-own-work-parent"); |
| 3669 | const projectId = ProjectId.make("project:followup-own-work"); |
| 3670 | const parentRunId = RunId.make("run:followup-own-work-parent"); |
| 3671 | const rootNodeId = NodeId.make("node:followup-own-work-parent"); |
| 3672 | yield* seedParentWithTerminalTask({ |
| 3673 | threadId: parentThreadId, |
| 3674 | projectId, |
| 3675 | runId: parentRunId, |
| 3676 | rootNodeId, |
| 3677 | taskId: NodeId.make("node:followup-own-work-seed"), |
| 3678 | deliveryState: "acknowledged", |
| 3679 | now, |
| 3680 | }); |
| 3681 | const child = yield* seedRestartCancelledChild({ |
| 3682 | parentThreadId, |
| 3683 | projectId, |
| 3684 | parentRunId, |
| 3685 | rootNodeId, |
| 3686 | name: "followup-own-work-child", |
| 3687 | completionWake: "always", |
| 3688 | continuationPending: false, |
| 3689 | runStatus: "completed", |
| 3690 | now, |
| 3691 | }); |
| 3692 | yield* orchestrator.recoverDelegatedTasks; |
| 3693 | const parent = yield* orchestrator.getThreadProjection(parentThreadId); |
| 3694 | const childProviderThread = { |
| 3695 | ...parent.providerThreads[0]!, |
| 3696 | id: ProviderThreadId.make("provider-thread:followup-own-work-child"), |
| 3697 | appThreadId: child.childThreadId, |
| 3698 | }; |
| 3699 | const awaited = (taskId: string) => ({ |
| 3700 | taskId, |
| 3701 | kind: "command" as const, |
| 3702 | awaited: true as const, |
| 3703 | }); |
| 3704 | const roster = (name: string, runId: RunId, tasks: ReadonlyArray<string>) => ({ |
| 3705 | id: EventId.make(`event:followup-own-work-roster-${name}`), |
| 3706 | type: "provider-thread.updated" as const, |
| 3707 | threadId: child.childThreadId, |
| 3708 | runId, |
| 3709 | driver, |
| 3710 | providerInstanceId: modelSelection.instanceId, |
| 3711 | occurredAt: now, |
| 3712 | payload: { ...childProviderThread, pendingBackgroundTasks: tasks.map(awaited) }, |
| 3713 | }); |
| 3714 | const settledTask = (taskId: NodeId, afterSequence: number) => |
| 3715 | eventSink.stream({ afterSequence, eventType: "subagent.updated" }).pipe( |
| 3716 | Stream.filter( |
| 3717 | (stored) => |
| 3718 | stored.event.type === "subagent.updated" && |
| 3719 | stored.event.payload.id === taskId && |
| 3720 | stored.event.payload.status === "completed", |
| 3721 | ), |
| 3722 | Stream.take(1), |
| 3723 | Stream.runDrain, |
| 3724 | ); |
| 3725 | const send = (name: string) => |
| 3726 | Effect.gen(function* () { |
| 3727 | const messageId = MessageId.make(`message:followup-own-work-send-${name}`); |
| 3728 | yield* orchestrator.dispatch({ |
| 3729 | type: "message.dispatch", |
| 3730 | createdBy: "agent", |
| 3731 | creationSource: "mcp", |
| 3732 | commandId: CommandId.make(`command:followup-own-work-send-${name}`), |
| 3733 | threadId: child.childThreadId, |
| 3734 | senderThreadId: parentThreadId, |
| 3735 | messageId, |
| 3736 | text: `Start build ${name} in the background and wait on it.`, |
| 3737 | attachments: [], |
| 3738 | dispatchMode: { type: "queue_after_active" }, |
| 3739 | }); |
| 3740 | return (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find( |
| 3741 | (run) => run.userMessageId === messageId, |
| 3742 | )!; |
| 3743 | }); |
| 3744 | |
| 3745 | // Follow-up A starts awaited build a, answers and ends its turn. |
| 3746 | const runA = yield* send("a"); |
| 3747 | yield* eventSink.write({ |
| 3748 | commandId: reconcileCommandId("followup-own-work-a-ended"), |
| 3749 | events: [ |
| 3750 | roster("a", runA.id, ["a"]), |
| 3751 | { |
| 3752 | id: EventId.make("event:followup-own-work-a-answer"), |
| 3753 | type: "message.updated", |
| 3754 | threadId: child.childThreadId, |
| 3755 | runId: runA.id, |
| 3756 | occurredAt: now, |
| 3757 | payload: { |
| 3758 | id: MessageId.make("message:followup-own-work-a-answer"), |
| 3759 | threadId: child.childThreadId, |
| 3760 | runId: runA.id, |
| 3761 | nodeId: null, |
| 3762 | role: "assistant", |
| 3763 | text: "A-DONE", |
| 3764 | attachments: [], |
| 3765 | streaming: false, |
| 3766 | createdBy: "agent", |
| 3767 | creationSource: "server", |
| 3768 | createdAt: now, |
| 3769 | updatedAt: now, |
| 3770 | }, |
| 3771 | }, |
| 3772 | { |
| 3773 | id: EventId.make("event:followup-own-work-a-completed"), |
| 3774 | type: "run.updated", |
| 3775 | threadId: child.childThreadId, |
| 3776 | runId: runA.id, |
| 3777 | occurredAt: now, |
| 3778 | payload: { ...runA, status: "completed", startedAt: now, completedAt: now }, |
| 3779 | }, |
| 3780 | ], |
| 3781 | }); |
| 3782 | yield* orchestrator.recoverDelegatedTasks; |
| 3783 | |
| 3784 | // Follow-up B starts awaited build b and is still running. |
| 3785 | const runB = yield* send("b"); |
| 3786 | yield* eventSink.write({ |
| 3787 | commandId: reconcileCommandId("followup-own-work-b-started"), |
| 3788 | events: [ |
| 3789 | { |
| 3790 | id: EventId.make("event:followup-own-work-b-running"), |
| 3791 | type: "run.updated", |
| 3792 | threadId: child.childThreadId, |
| 3793 | runId: runB.id, |
| 3794 | occurredAt: now, |
| 3795 | payload: { ...runB, status: "running", startedAt: now }, |
| 3796 | }, |
| 3797 | roster("ab", runB.id, ["a", "b"]), |
| 3798 | ], |
| 3799 | }); |
| 3800 | // Run updates react in order, so the barrier settling means B's start was handled too. |
| 3801 | const barrier = yield* seedRestartCancelledChild({ |
| 3802 | parentThreadId, |
| 3803 | projectId, |
| 3804 | parentRunId, |
| 3805 | rootNodeId, |
| 3806 | name: "followup-own-work-barrier", |
| 3807 | completionWake: "always", |
| 3808 | continuationPending: false, |
| 3809 | runStatus: "completed", |
| 3810 | now, |
| 3811 | }); |
| 3812 | const beforeBarrier = yield* eventSink.latestSequence(); |
| 3813 | yield* eventSink.write({ |
| 3814 | events: [ |
| 3815 | { |
| 3816 | ...runEvent({ |
| 3817 | threadId: barrier.childThreadId, |
| 3818 | runId: barrier.childRunId, |
| 3819 | ordinal: 1, |
| 3820 | status: "completed", |
| 3821 | now, |
| 3822 | }), |
| 3823 | id: EventId.make("event:followup-own-work-barrier-ended"), |
| 3824 | }, |
| 3825 | ], |
| 3826 | }); |
| 3827 | yield* settledTask(barrier.taskId, beforeBarrier); |
| 3828 | assert.equal( |
| 3829 | (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( |
| 3830 | (task) => task.id === runA.delegatedTaskId, |
| 3831 | )?.status, |
| 3832 | "running", |
| 3833 | ); |
| 3834 | |
| 3835 | // Build a ends; the roster still holds b, and no run update follows. |
| 3836 | const beforeSettle = yield* eventSink.latestSequence(); |
| 3837 | yield* eventSink.write({ |
| 3838 | commandId: reconcileCommandId("followup-own-work-a-settled"), |
| 3839 | events: [roster("b", runB.id, ["b"])], |
| 3840 | }); |
| 3841 | yield* settledTask(runA.delegatedTaskId!, beforeSettle); |
| 3842 | const tasks = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents; |
| 3843 | assert.equal(tasks.find((task) => task.id === runA.delegatedTaskId)?.result, "A-DONE"); |
| 3844 | assert.equal(tasks.find((task) => task.id === runB.delegatedTaskId)?.status, "running"); |
| 3845 | }), |
| 3846 | ); |
| 3847 | |
| 3848 | it.effect("waits for a run the follow-up's child queued on its own thread", () => |
| 3849 | Effect.gen(function* () { |
| 3850 | const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; |
| 3851 | const followup = yield* startSelfQueuedFollowup("self-queued-followup"); |
| 3852 | yield* followup.finishRun("handoff", followup.followupRun, "HANDOFF-C"); |
| 3853 | const waiting = yield* followup.readFollowup; |
| 3854 | assert.isNull( |
| 3855 | waiting.result, |
| 3856 | "the follow-up task settled on its handoff turn while the run its child queued was pending", |
| 3857 | ); |
| 3858 | assert.notEqual(waiting.status, "completed"); |
| 3859 | |
| 3860 | yield* followup.finishRun("final", followup.selfQueuedRun, "FINAL-D"); |
| 3861 | const finished = yield* followup.readFollowup; |
| 3862 | assert.equal(finished.status, "completed"); |
| 3863 | assert.equal(finished.result, "FINAL-D"); |
| 3864 | assert.equal(finished.completionDelivery?.state, "claimed"); |
| 3865 | let wake = yield* requests.take; |
| 3866 | while ( |
| 3867 | wake.threadId !== followup.parentThreadId || |
| 3868 | wake.delegatedCompletion?.messageId === followup.firstWake.delegatedCompletion?.messageId |
| 3869 | ) { |
| 3870 | wake = yield* requests.take; |
| 3871 | } |
| 3872 | assert.equal(wake.delegatedCompletion?.parentRunId, followup.parentRunId); |
| 3873 | }), |
| 3874 | ); |
| 3875 | |
| 3876 | it.effect("keeps a follow-up running when the run its child queued is removed early", () => |
| 3877 | Effect.gen(function* () { |
| 3878 | const followup = yield* startSelfQueuedFollowup("self-queued-removed-early-followup"); |
| 3879 | yield* followup.removeSelfQueuedRun; |
| 3880 | const running = yield* followup.readFollowup; |
| 3881 | assert.isNull(running.result); |
| 3882 | assert.equal(running.status, "running"); |
| 3883 | |
| 3884 | yield* followup.finishRun("handoff", followup.followupRun, "HANDOFF-C"); |
| 3885 | const finished = yield* followup.readFollowup; |
| 3886 | assert.equal(finished.status, "completed"); |
| 3887 | assert.equal(finished.result, "HANDOFF-C"); |
| 3888 | }), |
| 3889 | ); |
| 3890 | |
| 3891 | it.effect.each(["removed", "held"] as const)( |
| 3892 | "settles a follow-up on its last turn when the run it queued is %s", |
| 3893 | (ending) => |
| 3894 | Effect.gen(function* () { |
| 3895 | const eventSink = yield* EventSink.EventSinkV2; |
| 3896 | const now = yield* DateTime.now; |
| 3897 | const followup = yield* startSelfQueuedFollowup(`self-queued-${ending}-followup`); |
| 3898 | if (ending === "held") { |
| 3899 | yield* eventSink.write({ |
| 3900 | commandId: reconcileCommandId("self-queued-held-followup-hold"), |
| 3901 | events: [ |
| 3902 | { |
| 3903 | id: EventId.make("event:self-queued-held-followup-hold"), |
| 3904 | type: "run.updated", |
| 3905 | threadId: followup.childThreadId, |
| 3906 | runId: followup.selfQueuedRun.id, |
| 3907 | occurredAt: now, |
| 3908 | payload: { ...followup.selfQueuedRun, queueHeld: true }, |
| 3909 | }, |
| 3910 | ], |
| 3911 | }); |
| 3912 | } |
| 3913 | yield* followup.finishRun("handoff", followup.followupRun, "HANDOFF-C"); |
| 3914 | if (ending === "removed") yield* followup.removeSelfQueuedRun; |
| 3915 | const finished = yield* followup.readFollowup; |
| 3916 | assert.equal(finished.status, "completed"); |
| 3917 | assert.equal(finished.result, "HANDOFF-C"); |
| 3918 | }), |
| 3919 | ); |
| 3920 | |
| 3921 | it.effect("waits for every run its child queued when a later one is moved ahead", () => |
| 3922 | Effect.gen(function* () { |
| 3923 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 3924 | const followup = yield* startSelfQueuedFollowup("self-queued-reordered-followup"); |
| 3925 | const laterMessageId = MessageId.make("message:self-queued-reordered-followup-later"); |
| 3926 | yield* orchestrator.dispatch({ |
| 3927 | type: "message.dispatch", |
| 3928 | createdBy: "agent", |
| 3929 | creationSource: "mcp", |
| 3930 | commandId: CommandId.make("command:self-queued-reordered-followup-later"), |
| 3931 | threadId: followup.childThreadId, |
| 3932 | senderThreadId: followup.childThreadId, |
| 3933 | messageId: laterMessageId, |
| 3934 | text: "Do this part first.", |
| 3935 | attachments: [], |
| 3936 | dispatchMode: { type: "queue_after_active" }, |
| 3937 | }); |
| 3938 | const later = (yield* orchestrator.getThreadProjection(followup.childThreadId)).runs.find( |
| 3939 | (run) => run.userMessageId === laterMessageId, |
| 3940 | )!; |
| 3941 | yield* orchestrator.dispatch({ |
| 3942 | type: "queued-run.reorder", |
| 3943 | commandId: CommandId.make("command:self-queued-reordered-followup-reorder"), |
| 3944 | threadId: followup.childThreadId, |
| 3945 | runId: later.id, |
| 3946 | beforeRunId: followup.selfQueuedRun.id, |
| 3947 | }); |
| 3948 | assert.equal((yield* followup.readFollowup).status, "running"); |
| 3949 | yield* followup.finishRun("handoff", followup.followupRun, "HANDOFF-C"); |
| 3950 | yield* followup.finishRun("later", later, "PART-ONE"); |
| 3951 | const waiting = yield* followup.readFollowup; |
| 3952 | assert.isNull(waiting.result, "the follow-up settled while an earlier queued run waited"); |
| 3953 | |
| 3954 | // Promotion moves a queued run after the runs that started ahead of it. |
| 3955 | const promoted = { ...followup.selfQueuedRun, ordinal: later.ordinal + 1 }; |
| 3956 | yield* followup.finishRun("final", promoted, "FINAL-D"); |
| 3957 | const finished = yield* followup.readFollowup; |
| 3958 | assert.equal(finished.status, "completed"); |
| 3959 | assert.equal(finished.result, "FINAL-D"); |
| 3960 | }), |
| 3961 | ); |
| 3962 | |
| 3963 | const outcomes = [ |
| 3964 | "completed", |
| 3965 | "failed", |
| 3966 | "interrupted", |
| 3967 | "restarted", |
| 3968 | "stopped", |
| 3969 | "overlapping", |
| 3970 | "after-cancel", |
| 3971 | "cancelled", |
| 3972 | ] as const; |
| 3973 | it.effect.each(outcomes)( |
| 3974 | "delivers a %s follow-up independently of the original task", |
| 3975 | (outcome) => |
| 3976 | Effect.gen(function* () { |
| 3977 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 3978 | const eventSink = yield* EventSink.EventSinkV2; |
| 3979 | const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; |
| 3980 | const now = yield* DateTime.now; |
| 3981 | const parentThreadId = ThreadId.make(`thread:followup-${outcome}`); |
| 3982 | const projectId = ProjectId.make(`project:followup-${outcome}`); |
| 3983 | const parentRunId = RunId.make(`run:followup-${outcome}`); |
| 3984 | const rootNodeId = NodeId.make(`node:followup-${outcome}`); |
| 3985 | yield* seedParentWithTerminalTask({ |
| 3986 | threadId: parentThreadId, |
| 3987 | projectId, |
| 3988 | runId: parentRunId, |
| 3989 | rootNodeId, |
| 3990 | taskId: NodeId.make(`node:seed-${outcome}`), |
| 3991 | deliveryState: "acknowledged", |
| 3992 | now, |
| 3993 | }); |
| 3994 | const child = yield* seedRestartCancelledChild({ |
| 3995 | parentThreadId, |
| 3996 | projectId, |
| 3997 | parentRunId, |
| 3998 | rootNodeId, |
| 3999 | name: `followup-child-${outcome}`, |
| 4000 | completionWake: "always", |
| 4001 | continuationPending: false, |
| 4002 | runStatus: "completed", |
| 4003 | now, |
| 4004 | }); |
| 4005 | yield* orchestrator.recoverDelegatedTasks; |
| 4006 | let firstWake = yield* requests.take; |
| 4007 | while (firstWake.threadId !== parentThreadId) firstWake = yield* requests.take; |
| 4008 | assert.equal(firstWake.delegatedCompletion?.parentRunId, parentRunId); |
| 4009 | yield* orchestrator.dispatch({ |
| 4010 | type: "delegated_task.completion-delivery.acknowledge", |
| 4011 | commandId: CommandId.make(`command:followup-ack-${outcome}`), |
| 4012 | parentThreadId, |
| 4013 | taskId: child.taskId, |
| 4014 | observedByRunId: parentRunId, |
| 4015 | }); |
| 4016 | if (outcome === "after-cancel") { |
| 4017 | // A follow-up is its own task, even after task_cancel reached the original. |
| 4018 | const settled = (yield* orchestrator.getThreadProjection( |
| 4019 | parentThreadId, |
| 4020 | )).subagents.find((task) => task.id === child.taskId)!; |
| 4021 | // The follow-up starts in the same millisecond the cancel was recorded. |
| 4022 | yield* eventSink.write({ |
| 4023 | commandId: reconcileCommandId("followup-original-cancelled"), |
| 4024 | events: [ |
| 4025 | { |
| 4026 | id: EventId.make("event:followup-original-cancelled"), |
| 4027 | type: "subagent.updated", |
| 4028 | threadId: parentThreadId, |
| 4029 | nodeId: settled.id, |
| 4030 | occurredAt: now, |
| 4031 | payload: { ...settled, cancelledRunId: child.childRunId }, |
| 4032 | }, |
| 4033 | ], |
| 4034 | }); |
| 4035 | } |
| 4036 | const original = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( |
| 4037 | (task) => task.id === child.taskId, |
| 4038 | )!; |
| 4039 | const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.find( |
| 4040 | (run) => run.id === parentRunId, |
| 4041 | )!; |
| 4042 | yield* eventSink.write({ |
| 4043 | commandId: reconcileCommandId(`followup-parent-finished-${outcome}`), |
| 4044 | events: [ |
| 4045 | { |
| 4046 | id: EventId.make(`event:followup-parent-finished-${outcome}`), |
| 4047 | type: "run.updated", |
| 4048 | threadId: parentThreadId, |
| 4049 | runId: parentRunId, |
| 4050 | occurredAt: now, |
| 4051 | payload: { ...parentRun, status: "completed", completedAt: now }, |
| 4052 | }, |
| 4053 | ], |
| 4054 | }); |
| 4055 | const messageId = MessageId.make(`message:followup-${outcome}`); |
| 4056 | yield* orchestrator.dispatch({ |
| 4057 | type: "message.dispatch", |
| 4058 | createdBy: outcome === "failed" ? "user" : "agent", |
| 4059 | creationSource: outcome === "failed" ? "web" : "mcp", |
| 4060 | commandId: CommandId.make(`command:followup-send-${outcome}`), |
| 4061 | threadId: child.childThreadId, |
| 4062 | // `t3 thread send` and unscoped MCP sessions carry no sender thread. |
| 4063 | ...(outcome === "completed" ? {} : { senderThreadId: parentThreadId }), |
| 4064 | messageId, |
| 4065 | text: "Check the same work again.", |
| 4066 | attachments: [], |
| 4067 | dispatchMode: { type: "start_immediately" }, |
| 4068 | }); |
| 4069 | const childProjection = yield* orchestrator.getThreadProjection(child.childThreadId); |
| 4070 | let run = childProjection.runs.find((row) => row.userMessageId === messageId)!; |
| 4071 | assert.isDefined(run.delegatedTaskId); |
| 4072 | const workingParent = yield* orchestrator.getShellSnapshot(); |
| 4073 | const backgroundTask = workingParent.threads |
| 4074 | .find((thread) => thread.id === parentThreadId) |
| 4075 | ?.pendingBackgroundTasks?.find((task) => task.kind === "subagent"); |
| 4076 | assert.equal(backgroundTask?.childThreadId, child.childThreadId); |
| 4077 | if (outcome === "overlapping") { |
| 4078 | yield* orchestrator.dispatch({ |
| 4079 | type: "message.dispatch", |
| 4080 | createdBy: "agent", |
| 4081 | creationSource: "mcp", |
| 4082 | commandId: CommandId.make("command:overlapping-followup"), |
| 4083 | threadId: child.childThreadId, |
| 4084 | senderThreadId: parentThreadId, |
| 4085 | messageId: MessageId.make("message:overlapping-followup"), |
| 4086 | text: "Keep working on another follow-up.", |
| 4087 | attachments: [], |
| 4088 | dispatchMode: { type: "queue_after_active" }, |
| 4089 | }); |
| 4090 | } |
| 4091 | if (outcome === "restarted") { |
| 4092 | const cut = { ...run, status: "cancelled" as const, startedAt: now, completedAt: now }; |
| 4093 | yield* eventSink.writeWithEffects({ |
| 4094 | commandId: reconcileCommandId("followup-restart-cut"), |
| 4095 | events: [ |
| 4096 | { |
| 4097 | id: EventId.make("event:followup-restart-cut"), |
| 4098 | type: "run.updated", |
| 4099 | threadId: child.childThreadId, |
| 4100 | runId: run.id, |
| 4101 | occurredAt: now, |
| 4102 | payload: cut, |
| 4103 | }, |
| 4104 | ], |
| 4105 | effects: [ |
| 4106 | { |
| 4107 | id: `effect:restart-continuation:${run.id}`, |
| 4108 | commandId: reconcileCommandId("followup-restart-cut"), |
| 4109 | threadId: child.childThreadId, |
| 4110 | request: { type: "provider-runtime.continue", sourceRunId: run.id }, |
| 4111 | }, |
| 4112 | ], |
| 4113 | }); |
| 4114 | yield* orchestrator.recoverDelegatedTasks; |
| 4115 | assert.isNull( |
| 4116 | (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( |
| 4117 | (task) => task.id === run.delegatedTaskId, |
| 4118 | )!.result, |
| 4119 | ); |
| 4120 | const continuationMessageId = MessageId.make("message:followup-restart-continuation"); |
| 4121 | yield* orchestrator.dispatch({ |
| 4122 | type: "message.dispatch", |
| 4123 | createdBy: "agent", |
| 4124 | creationSource: "server", |
| 4125 | commandId: CommandId.make("command:followup-restart-continuation"), |
| 4126 | threadId: child.childThreadId, |
| 4127 | restartContinuationOfRunId: run.id, |
| 4128 | messageId: continuationMessageId, |
| 4129 | text: "Continue where you left off.", |
| 4130 | attachments: [], |
| 4131 | dispatchMode: { type: "start_immediately" }, |
| 4132 | }); |
| 4133 | const continuation = (yield* orchestrator.getThreadProjection( |
| 4134 | child.childThreadId, |
| 4135 | )).runs.find((row) => row.userMessageId === continuationMessageId)!; |
| 4136 | assert.equal(continuation.delegatedTaskId, run.delegatedTaskId); |
| 4137 | run = continuation; |
| 4138 | } |
| 4139 | if (outcome === "stopped") { |
| 4140 | yield* orchestrator.dispatch({ |
| 4141 | type: "thread.stop", |
| 4142 | threadId: parentThreadId, |
| 4143 | commandId: CommandId.make(`command:followup-stop-${outcome}`), |
| 4144 | }); |
| 4145 | } |
| 4146 | if (outcome === "cancelled") { |
| 4147 | // task_cancel reaches the follow-up after its turn ended, while background work runs. |
| 4148 | yield* eventSink.write({ |
| 4149 | commandId: reconcileCommandId("followup-waiting"), |
| 4150 | events: [ |
| 4151 | { |
| 4152 | id: EventId.make("event:followup-waiting"), |
| 4153 | type: "run.updated", |
| 4154 | threadId: child.childThreadId, |
| 4155 | runId: run.id, |
| 4156 | occurredAt: now, |
| 4157 | payload: { ...run, status: "waiting", startedAt: now }, |
| 4158 | }, |
| 4159 | ], |
| 4160 | }); |
| 4161 | yield* orchestrator.cancelDelegatedTask({ |
| 4162 | parentThreadId, |
| 4163 | taskId: run.delegatedTaskId!, |
| 4164 | command: { |
| 4165 | type: "thread.stop", |
| 4166 | commandId: CommandId.make(`cancel-followup-${outcome}`), |
| 4167 | threadId: child.childThreadId, |
| 4168 | stopDelegatedTasks: false, |
| 4169 | }, |
| 4170 | stopDescendants: () => Effect.void, |
| 4171 | }); |
| 4172 | } |
| 4173 | yield* eventSink.write({ |
| 4174 | commandId: reconcileCommandId(`followup-finish-${outcome}`), |
| 4175 | events: [ |
| 4176 | { |
| 4177 | id: EventId.make(`event:followup-question-${outcome}`), |
| 4178 | type: "runtime-request.updated", |
| 4179 | threadId: child.childThreadId, |
| 4180 | runId: run.id, |
| 4181 | occurredAt: now, |
| 4182 | payload: { |
| 4183 | id: RuntimeRequestId.make(`request:followup-${outcome}`), |
| 4184 | nodeId: run.rootNodeId!, |
| 4185 | providerTurnId: null, |
| 4186 | nativeRequestRef: null, |
| 4187 | kind: "user_input", |
| 4188 | status: "pending", |
| 4189 | responseCapability: { type: "message" }, |
| 4190 | createdAt: now, |
| 4191 | resolvedAt: null, |
| 4192 | }, |
| 4193 | }, |
| 4194 | { |
| 4195 | id: EventId.make(`event:followup-finish-${outcome}`), |
| 4196 | type: "run.updated", |
| 4197 | threadId: child.childThreadId, |
| 4198 | runId: run.id, |
| 4199 | occurredAt: now, |
| 4200 | payload: { |
| 4201 | ...run, |
| 4202 | status: |
| 4203 | outcome === "stopped" |
| 4204 | ? "interrupted" |
| 4205 | : outcome === "restarted" || |
| 4206 | outcome === "overlapping" || |
| 4207 | outcome === "after-cancel" || |
| 4208 | outcome === "cancelled" |
| 4209 | ? "completed" |
| 4210 | : outcome, |
| 4211 | startedAt: now, |
| 4212 | completedAt: now, |
| 4213 | }, |
| 4214 | }, |
| 4215 | ], |
| 4216 | }); |
| 4217 | yield* orchestrator.recoverDelegatedTasks; |
| 4218 | // Only an interrupted or cancelled follow-up ends its question, even after the original |
| 4219 | // was cancelled. |
| 4220 | const ended = |
| 4221 | outcome === "stopped" || outcome === "interrupted" || outcome === "cancelled"; |
| 4222 | const { runtimeRequests } = yield* orchestrator.getThreadProjection(child.childThreadId); |
| 4223 | assert.equal(runtimeRequests[0]?.status, ended ? "resolved" : "pending"); |
| 4224 | const finished = yield* orchestrator.getThreadProjection(parentThreadId); |
| 4225 | assert.deepEqual( |
| 4226 | finished.subagents.find((task) => task.id === child.taskId), |
| 4227 | original, |
| 4228 | ); |
| 4229 | const followup = finished.subagents.find((task) => task.id === run.delegatedTaskId)!; |
| 4230 | if (outcome === "overlapping") { |
| 4231 | const later = finished.subagents.find( |
| 4232 | (task) => |
| 4233 | task.childThreadId === child.childThreadId && |
| 4234 | task.id !== child.taskId && |
| 4235 | task.id !== followup.id, |
| 4236 | )!; |
| 4237 | assert.equal(later.status, "pending"); |
| 4238 | assert.isNull(later.result); |
| 4239 | } |
| 4240 | assert.equal( |
| 4241 | followup.status, |
| 4242 | outcome === "stopped" || outcome === "cancelled" |
| 4243 | ? "interrupted" |
| 4244 | : outcome === "restarted" || outcome === "overlapping" || outcome === "after-cancel" |
| 4245 | ? "completed" |
| 4246 | : outcome, |
| 4247 | ); |
| 4248 | assert.equal( |
| 4249 | followup.completionDelivery?.state, |
| 4250 | outcome === "stopped" || outcome === "cancelled" ? "disposed" : "claimed", |
| 4251 | ); |
| 4252 | assert.equal(followup.cancelledRunId, outcome === "cancelled" ? run.id : undefined); |
| 4253 | if (outcome !== "stopped" && outcome !== "cancelled") { |
| 4254 | let wake = yield* requests.take; |
| 4255 | while ( |
| 4256 | wake.threadId !== parentThreadId || |
| 4257 | wake.delegatedCompletion?.messageId === firstWake.delegatedCompletion?.messageId |
| 4258 | ) { |
| 4259 | wake = yield* requests.take; |
| 4260 | } |
| 4261 | assert.equal(wake.delegatedCompletion?.parentRunId, parentRunId); |
| 4262 | assert.notEqual( |
| 4263 | wake.delegatedCompletion?.messageId, |
| 4264 | firstWake.delegatedCompletion?.messageId, |
| 4265 | ); |
| 4266 | } |
| 4267 | assert.equal( |
| 4268 | finished.contextTransfers.filter( |
| 4269 | (transfer) => |
| 4270 | transfer.type === "subagent_result" && |
| 4271 | transfer.sourceThreadId === child.childThreadId, |
| 4272 | ).length, |
| 4273 | 2, |
| 4274 | ); |
| 4275 | yield* orchestrator.recoverDelegatedTasks; |
| 4276 | const recovered = yield* orchestrator.getThreadProjection(parentThreadId); |
| 4277 | assert.equal(recovered.contextTransfers.length, finished.contextTransfers.length); |
| 4278 | if (outcome === "completed") { |
| 4279 | const automaticMessageId = MessageId.make("message:followup-automatic-wake"); |
| 4280 | yield* orchestrator.dispatch({ |
| 4281 | type: "message.dispatch", |
| 4282 | createdBy: "agent", |
| 4283 | creationSource: "server", |
| 4284 | commandId: CommandId.make("command:followup-automatic-wake"), |
| 4285 | threadId: child.childThreadId, |
| 4286 | messageId: automaticMessageId, |
| 4287 | text: "A monitor reported new output.", |
| 4288 | attachments: [], |
| 4289 | notification: { |
| 4290 | source: { kind: "monitor" }, |
| 4291 | outcome: "updated", |
| 4292 | summary: "Monitor updated", |
| 4293 | }, |
| 4294 | dispatchMode: { type: "queue_after_active" }, |
| 4295 | }); |
| 4296 | const automatic = (yield* orchestrator.getThreadProjection( |
| 4297 | child.childThreadId, |
| 4298 | )).runs.find((run) => run.userMessageId === automaticMessageId)!; |
| 4299 | assert.isUndefined(automatic.delegatedTaskId); |
| 4300 | yield* eventSink.write({ |
| 4301 | commandId: reconcileCommandId("followup-automatic-finished"), |
| 4302 | events: [ |
| 4303 | { |
| 4304 | id: EventId.make("event:followup-automatic-finished"), |
| 4305 | type: "run.updated", |
| 4306 | threadId: child.childThreadId, |
| 4307 | runId: automatic.id, |
| 4308 | occurredAt: now, |
| 4309 | payload: { ...automatic, status: "completed", startedAt: now, completedAt: now }, |
| 4310 | }, |
| 4311 | ], |
| 4312 | }); |
| 4313 | yield* orchestrator.recoverDelegatedTasks; |
| 4314 | const afterAutomatic = yield* orchestrator.getThreadProjection(parentThreadId); |
| 4315 | assert.deepEqual(afterAutomatic.subagents, recovered.subagents); |
| 4316 | assert.equal(afterAutomatic.contextTransfers.length, recovered.contextTransfers.length); |
| 4317 | const control = yield* seedRestartCancelledChild({ |
| 4318 | parentThreadId, |
| 4319 | projectId, |
| 4320 | parentRunId, |
| 4321 | rootNodeId, |
| 4322 | name: "followup-control-child", |
| 4323 | completionWake: "always", |
| 4324 | continuationPending: false, |
| 4325 | runStatus: "completed", |
| 4326 | now, |
| 4327 | }); |
| 4328 | yield* orchestrator.recoverDelegatedTasks; |
| 4329 | const afterControl = yield* orchestrator.getThreadProjection(parentThreadId); |
| 4330 | assert.equal( |
| 4331 | afterControl.subagents.find((task) => task.id === control.taskId)?.completionDelivery |
| 4332 | ?.state, |
| 4333 | "claimed", |
| 4334 | ); |
| 4335 | assert.equal( |
| 4336 | afterControl.contextTransfers.filter( |
| 4337 | (transfer) => |
| 4338 | transfer.type === "subagent_result" && |
| 4339 | transfer.sourceThreadId === control.childThreadId, |
| 4340 | ).length, |
| 4341 | 1, |
| 4342 | ); |
| 4343 | } |
| 4344 | }), |
| 4345 | ); |
| 4346 | |
| 4347 | it.effect("tracks a child's usage-limit auto-resume as a follow-up", () => |
| 4348 | Effect.gen(function* () { |
| 4349 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 4350 | const eventSink = yield* EventSink.EventSinkV2; |
| 4351 | const now = yield* DateTime.now; |
| 4352 | const parentThreadId = ThreadId.make("thread:limit-resume-parent"); |
| 4353 | const projectId = ProjectId.make("project:limit-resume"); |
| 4354 | const parentRunId = RunId.make("run:limit-resume-parent"); |
| 4355 | const rootNodeId = NodeId.make("node:limit-resume-parent"); |
| 4356 | yield* seedParentWithTerminalTask({ |
| 4357 | threadId: parentThreadId, |
| 4358 | projectId, |
| 4359 | runId: parentRunId, |
| 4360 | rootNodeId, |
| 4361 | taskId: NodeId.make("node:limit-resume-seed"), |
| 4362 | deliveryState: "acknowledged", |
| 4363 | now, |
| 4364 | }); |
| 4365 | const child = yield* seedRestartCancelledChild({ |
| 4366 | parentThreadId, |
| 4367 | projectId, |
| 4368 | parentRunId, |
| 4369 | rootNodeId, |
| 4370 | name: "limit-resume-child", |
| 4371 | completionWake: "always", |
| 4372 | continuationPending: false, |
| 4373 | runStatus: "completed", |
| 4374 | now, |
| 4375 | }); |
| 4376 | const childRun = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs[0]!; |
| 4377 | yield* eventSink.write({ |
| 4378 | commandId: reconcileCommandId("limit-resume-child-limited"), |
| 4379 | events: [ |
| 4380 | { |
| 4381 | id: EventId.make("event:limit-resume-child-failed"), |
| 4382 | type: "run.updated", |
| 4383 | threadId: child.childThreadId, |
| 4384 | runId: childRun.id, |
| 4385 | occurredAt: now, |
| 4386 | payload: { ...childRun, status: "failed", completedAt: now }, |
| 4387 | }, |
| 4388 | { |
| 4389 | id: EventId.make("event:limit-resume-child-error"), |
| 4390 | type: "turn-item.updated", |
| 4391 | threadId: child.childThreadId, |
| 4392 | occurredAt: now, |
| 4393 | payload: { |
| 4394 | id: TurnItemId.make("turn-item:limit-resume-child-error"), |
| 4395 | type: "error", |
| 4396 | threadId: child.childThreadId, |
| 4397 | runId: childRun.id, |
| 4398 | nodeId: childRun.rootNodeId, |
| 4399 | providerThreadId: null, |
| 4400 | providerTurnId: null, |
| 4401 | nativeItemRef: null, |
| 4402 | parentItemId: null, |
| 4403 | ordinal: 1, |
| 4404 | status: "failed", |
| 4405 | title: "Usage limit reached", |
| 4406 | startedAt: now, |
| 4407 | completedAt: now, |
| 4408 | updatedAt: now, |
| 4409 | failure: { |
| 4410 | class: "usage_limit", |
| 4411 | message: "Plan limit reached.", |
| 4412 | code: "usageLimitExceeded", |
| 4413 | retryable: null, |
| 4414 | resetAt: DateTime.formatIso(DateTime.add(now, { minutes: 1 })), |
| 4415 | }, |
| 4416 | }, |
| 4417 | }, |
| 4418 | ], |
| 4419 | }); |
| 4420 | yield* orchestrator.recoverDelegatedTasks; |
| 4421 | const childShell = Effect.map(orchestrator.getShellSnapshot(), (snapshot) => |
| 4422 | snapshot.threads.find((thread) => thread.id === child.childThreadId), |
| 4423 | ); |
| 4424 | const arm = limitRecoveryCommand((yield* childShell)!, true, DateTime.toEpochMillis(now)); |
| 4425 | assert(arm?.type === "thread.metadata.update" && arm.limitRecovery != null); |
| 4426 | yield* orchestrator.autoArmLimitRecovery({ |
| 4427 | commandId: arm.commandId, |
| 4428 | threadId: arm.threadId, |
| 4429 | limitRecovery: arm.limitRecovery, |
| 4430 | }); |
| 4431 | yield* TestClock.adjust("1 minute"); |
| 4432 | const resume = limitRecoveryCommand( |
| 4433 | (yield* childShell)!, |
| 4434 | true, |
| 4435 | DateTime.toEpochMillis(yield* DateTime.now), |
| 4436 | ); |
| 4437 | assert(resume?.type === "message.dispatch"); |
| 4438 | yield* orchestrator.dispatch(resume); |
| 4439 | |
| 4440 | const resumed = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find( |
| 4441 | (run) => run.userMessageId === resume.messageId, |
| 4442 | ); |
| 4443 | assert.isDefined(resumed?.delegatedTaskId); |
| 4444 | const followup = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( |
| 4445 | (task) => task.id === resumed?.delegatedTaskId, |
| 4446 | ); |
| 4447 | assert.equal(followup?.childThreadId, child.childThreadId); |
| 4448 | assert.isNull(followup?.result); |
| 4449 | }), |
| 4450 | ); |
| 4451 | |
| 4452 | it.effect("serializes a recovered child answer with its parent and tracks the follow-up", () => |
| 4453 | Effect.gen(function* () { |
| 4454 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 4455 | const eventSink = yield* EventSink.EventSinkV2; |
| 4456 | const now = yield* DateTime.now; |
| 4457 | const parentThreadId = ThreadId.make("thread:question-answer-parent"); |
| 4458 | const projectId = ProjectId.make("project:question-answer"); |
| 4459 | const parentRunId = RunId.make("run:question-answer-parent"); |
| 4460 | const rootNodeId = NodeId.make("node:question-answer-parent"); |
| 4461 | yield* seedParentWithTerminalTask({ |
| 4462 | threadId: parentThreadId, |
| 4463 | projectId, |
| 4464 | runId: parentRunId, |
| 4465 | rootNodeId, |
| 4466 | taskId: NodeId.make("node:question-answer-seed"), |
| 4467 | deliveryState: "acknowledged", |
| 4468 | now, |
| 4469 | }); |
| 4470 | const child = yield* seedRestartCancelledChild({ |
| 4471 | parentThreadId, |
| 4472 | projectId, |
| 4473 | parentRunId, |
| 4474 | rootNodeId, |
| 4475 | name: "question-answer-child", |
| 4476 | completionWake: "always", |
| 4477 | continuationPending: false, |
| 4478 | now, |
| 4479 | }); |
| 4480 | // The cancelled result is published before anyone answers the question. |
| 4481 | yield* orchestrator.recoverDelegatedTasks; |
| 4482 | assert.isNotNull( |
| 4483 | (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( |
| 4484 | (task) => task.id === child.taskId, |
| 4485 | )?.result, |
| 4486 | ); |
| 4487 | const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.find( |
| 4488 | (run) => run.id === parentRunId, |
| 4489 | )!; |
| 4490 | yield* eventSink.write({ |
| 4491 | events: [ |
| 4492 | { |
| 4493 | id: EventId.make("event:question-answer-parent-completed"), |
| 4494 | type: "run.updated", |
| 4495 | threadId: parentThreadId, |
| 4496 | runId: parentRunId, |
| 4497 | occurredAt: now, |
| 4498 | payload: { ...parentRun, status: "completed", completedAt: now }, |
| 4499 | }, |
| 4500 | ], |
| 4501 | }); |
| 4502 | const requestId = RuntimeRequestId.make("request:question-answer"); |
| 4503 | const nodeId = NodeId.make("node:question-answer-request"); |
| 4504 | yield* eventSink.write({ |
| 4505 | commandId: CommandId.make("command:seed-question-answer"), |
| 4506 | events: [ |
| 4507 | { |
| 4508 | id: EventId.make("event:question-answer-request"), |
| 4509 | type: "runtime-request.updated", |
| 4510 | threadId: child.childThreadId, |
| 4511 | nodeId, |
| 4512 | occurredAt: now, |
| 4513 | payload: { |
| 4514 | id: requestId, |
| 4515 | nodeId, |
| 4516 | providerTurnId: null, |
| 4517 | nativeRequestRef: null, |
| 4518 | kind: "user_input", |
| 4519 | status: "pending", |
| 4520 | responseCapability: { type: "message" }, |
| 4521 | createdAt: now, |
| 4522 | resolvedAt: null, |
| 4523 | }, |
| 4524 | }, |
| 4525 | { |
| 4526 | id: EventId.make("event:question-answer-item"), |
| 4527 | type: "turn-item.updated", |
| 4528 | threadId: child.childThreadId, |
| 4529 | nodeId, |
| 4530 | occurredAt: now, |
| 4531 | payload: { |
| 4532 | id: TurnItemId.make("turn-item:question-answer"), |
| 4533 | type: "user_input_request", |
| 4534 | threadId: child.childThreadId, |
| 4535 | runId: child.childRunId, |
| 4536 | nodeId, |
| 4537 | providerThreadId: null, |
| 4538 | providerTurnId: null, |
| 4539 | nativeItemRef: null, |
| 4540 | parentItemId: null, |
| 4541 | ordinal: 0, |
| 4542 | status: "waiting", |
| 4543 | title: null, |
| 4544 | startedAt: now, |
| 4545 | completedAt: null, |
| 4546 | updatedAt: now, |
| 4547 | requestId, |
| 4548 | questions: [ |
| 4549 | { id: "color", header: "Color", question: "Which color?", options: [] }, |
| 4550 | ], |
| 4551 | }, |
| 4552 | }, |
| 4553 | ], |
| 4554 | }); |
| 4555 | |
| 4556 | const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor; |
| 4557 | const parentHeld = yield* Deferred.make<void>(); |
| 4558 | const releaseParent = yield* Deferred.make<void>(); |
| 4559 | const answerWaiting = yield* Deferred.make<void>(); |
| 4560 | const withLock = executor.withLock; |
| 4561 | const parentMutation = yield* withLock( |
| 4562 | parentThreadId, |
| 4563 | Deferred.succeed(parentHeld, undefined).pipe( |
| 4564 | Effect.andThen(Deferred.await(releaseParent)), |
| 4565 | ), |
| 4566 | ).pipe(Effect.forkScoped({ startImmediately: true })); |
| 4567 | yield* Deferred.await(parentHeld); |
| 4568 | const observeLock: ThreadCommandExecutor.ThreadCommandExecutor["Service"]["withLock"] = ( |
| 4569 | key, |
| 4570 | effect, |
| 4571 | ) => |
| 4572 | key === parentThreadId |
| 4573 | ? Deferred.succeed(answerWaiting, undefined).pipe(Effect.andThen(withLock(key, effect))) |
| 4574 | : withLock(key, effect); |
| 4575 | const spy = vi.spyOn(executor, "withLock").mockImplementation(observeLock); |
| 4576 | yield* Effect.gen(function* () { |
| 4577 | const answer = yield* orchestrator |
| 4578 | .dispatch({ |
| 4579 | type: "runtime-request.respond", |
| 4580 | commandId: CommandId.make("command:question-answer"), |
| 4581 | threadId: child.childThreadId, |
| 4582 | requestId, |
| 4583 | answers: { color: "Blue" }, |
| 4584 | }) |
| 4585 | .pipe(Effect.forkScoped({ startImmediately: true })); |
| 4586 | yield* Effect.raceFirst(Deferred.await(answerWaiting), Fiber.join(answer)); |
| 4587 | const waiting = yield* orchestrator.getThreadProjection(child.childThreadId); |
| 4588 | assert.equal( |
| 4589 | waiting.runtimeRequests.find((row) => row.id === requestId)?.status, |
| 4590 | "pending", |
| 4591 | ); |
| 4592 | assert.isFalse((yield* executor.activeKeys).includes(child.childThreadId)); |
| 4593 | yield* Deferred.succeed(releaseParent, undefined); |
| 4594 | yield* Fiber.join(parentMutation); |
| 4595 | yield* Fiber.join(answer); |
| 4596 | }).pipe(Effect.ensuring(Effect.sync(() => spy.mockRestore()))); |
| 4597 | |
| 4598 | const answerRun = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find( |
| 4599 | (run) => run.userMessageId === MessageId.make(`async-answer:${requestId}`), |
| 4600 | ); |
| 4601 | assert.isDefined(answerRun?.delegatedTaskId); |
| 4602 | const followup = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( |
| 4603 | (task) => task.id === answerRun?.delegatedTaskId, |
| 4604 | ); |
| 4605 | assert.equal(followup?.childThreadId, child.childThreadId); |
| 4606 | assert.isNull(followup?.result); |
| 4607 | assert.isTrue( |
| 4608 | (yield* orchestrator.getThreadShell(parentThreadId))?.pendingBackgroundTasks?.some( |
| 4609 | (task) => task.kind === "subagent" && task.childThreadId === child.childThreadId, |
| 4610 | ), |
| 4611 | ); |
| 4612 | }), |
| 4613 | ); |
| 4614 | }, |
| 4615 | ); |
| 4616 | |
| 4617 | it.layer(layerTest)("delegated tasks across a server restart", (it) => { |
| 4618 | it.effect.each([ |
| 4619 | { name: "releases", key: "", pendingBackgroundTasks: [], released: true }, |
| 4620 | { |
| 4621 | name: "keeps while an unawaited background command runs", |
| 4622 | key: "-background", |
| 4623 | pendingBackgroundTasks: [{ taskId: "dev-server", kind: "command" as const }], |
| 4624 | released: false, |
| 4625 | }, |
| 4626 | ])( |
| 4627 | "queues a child result without creating a provider history handoff and $name its process", |
| 4628 | ({ key, pendingBackgroundTasks, released }) => |
| 4629 | Effect.gen(function* () { |
| 4630 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 4631 | const eventSink = yield* EventSink.EventSinkV2; |
| 4632 | const now = yield* DateTime.now; |
| 4633 | const threadId = ThreadId.make(`thread:result-message${key}-parent`); |
| 4634 | const projectId = ProjectId.make(`project:result-message${key}`); |
| 4635 | const runId = RunId.make(`run:result-message${key}-parent`); |
| 4636 | const rootNodeId = NodeId.make(`node:result-message${key}-parent`); |
| 4637 | yield* seedParentWithTerminalTask({ |
| 4638 | threadId, |
| 4639 | projectId, |
| 4640 | runId, |
| 4641 | rootNodeId, |
| 4642 | taskId: NodeId.make(`node:result-message${key}-seed`), |
| 4643 | deliveryState: "acknowledged", |
| 4644 | now, |
| 4645 | }); |
| 4646 | const child = yield* seedRestartCancelledChild({ |
| 4647 | parentThreadId: threadId, |
| 4648 | projectId, |
| 4649 | parentRunId: runId, |
| 4650 | rootNodeId, |
| 4651 | name: `result-message${key}-child`, |
| 4652 | completionWake: "always", |
| 4653 | continuationPending: false, |
| 4654 | runStatus: "completed", |
| 4655 | now, |
| 4656 | }); |
| 4657 | const parent = yield* orchestrator.getThreadProjection(threadId); |
| 4658 | const childProjection = yield* orchestrator.getThreadProjection(child.childThreadId); |
| 4659 | const providerThreadId = ProviderThreadId.make( |
| 4660 | `provider-thread:result-message${key}-child`, |
| 4661 | ); |
| 4662 | yield* eventSink.write({ |
| 4663 | commandId: CommandId.make(`command:result-message${key}-child-provider`), |
| 4664 | events: [ |
| 4665 | { |
| 4666 | id: EventId.make(`event:result-message${key}-child-session`), |
| 4667 | type: "provider-session.attached", |
| 4668 | threadId: child.childThreadId, |
| 4669 | occurredAt: now, |
| 4670 | payload: { |
| 4671 | id: ProviderSessionId.make(`provider-session:result-message${key}-child`), |
| 4672 | driver, |
| 4673 | providerInstanceId: modelSelection.instanceId, |
| 4674 | status: "ready", |
| 4675 | cwd: "/repo", |
| 4676 | model: modelSelection.model, |
| 4677 | capabilities: { |
| 4678 | ...CodexProviderCapabilitiesV2, |
| 4679 | sessions: { |
| 4680 | ...CodexProviderCapabilitiesV2.sessions, |
| 4681 | supportsMultipleProviderThreadsPerSession: false, |
| 4682 | }, |
| 4683 | }, |
| 4684 | createdAt: now, |
| 4685 | updatedAt: now, |
| 4686 | lastError: null, |
| 4687 | }, |
| 4688 | }, |
| 4689 | { |
| 4690 | id: EventId.make(`event:result-message${key}-child-provider`), |
| 4691 | type: "provider-thread.updated", |
| 4692 | threadId: child.childThreadId, |
| 4693 | occurredAt: now, |
| 4694 | payload: { |
| 4695 | ...parent.providerThreads[0]!, |
| 4696 | id: providerThreadId, |
| 4697 | appThreadId: child.childThreadId, |
| 4698 | pendingBackgroundTasks, |
| 4699 | }, |
| 4700 | }, |
| 4701 | { |
| 4702 | id: EventId.make(`event:result-message${key}-child-run`), |
| 4703 | type: "run.updated", |
| 4704 | threadId: child.childThreadId, |
| 4705 | runId: child.childRunId, |
| 4706 | occurredAt: now, |
| 4707 | payload: { ...childProjection.runs[0]!, providerThreadId }, |
| 4708 | }, |
| 4709 | ], |
| 4710 | }); |
| 4711 | |
| 4712 | yield* orchestrator.recoverDelegatedTasks; |
| 4713 | const recovered = yield* orchestrator.getThreadProjection(threadId); |
| 4714 | const task = recovered.subagents.find((candidate) => candidate.id === child.taskId)!; |
| 4715 | assert.equal(task.status, "completed"); |
| 4716 | assert.isNotNull(task.result); |
| 4717 | assert.isEmpty(recovered.contextHandoffs); |
| 4718 | assert.equal(recovered.contextTransfers.length, 1); |
| 4719 | // A one-process-per-thread child releases its process once its result is recorded, |
| 4720 | // unless background work it did not await would die with it. |
| 4721 | assert.equal( |
| 4722 | (yield* orchestrator.getThreadProjection(child.childThreadId)).providerSessions.length, |
| 4723 | released ? 0 : 1, |
| 4724 | ); |
| 4725 | |
| 4726 | const delivery = recovered.runs.find((candidate) => candidate.id === runId)! |
| 4727 | .delegatedCompletion!.delivery!; |
| 4728 | const messageId = delivery.messageId; |
| 4729 | yield* orchestrator.dispatch({ |
| 4730 | type: "message.dispatch", |
| 4731 | commandId: CommandId.make(`command:result-message${key}-delivery`), |
| 4732 | threadId, |
| 4733 | messageId, |
| 4734 | text: "Delegated task finished.", |
| 4735 | attachments: [], |
| 4736 | dispatchMode: { type: "queue_after_active" }, |
| 4737 | createdBy: "agent", |
| 4738 | creationSource: "server", |
| 4739 | delegatedCompletion: { |
| 4740 | parentRunId: runId, |
| 4741 | generation: delivery.generation, |
| 4742 | taskIds: [child.taskId], |
| 4743 | }, |
| 4744 | }); |
| 4745 | const queued = yield* orchestrator.getThreadProjection(threadId); |
| 4746 | assert.include( |
| 4747 | queued.messages.find((message) => message.id === messageId)?.text ?? "", |
| 4748 | task.result!, |
| 4749 | ); |
| 4750 | }), |
| 4751 | ); |
| 4752 | |
| 4753 | it.effect("holds a restart-cancelled child for its continuation's result", () => |
| 4754 | Effect.gen(function* () { |
| 4755 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 4756 | const eventSink = yield* EventSink.EventSinkV2; |
| 4757 | const now = yield* DateTime.now; |
| 4758 | const threadId = ThreadId.make("thread:restart-parent"); |
| 4759 | const projectId = ProjectId.make("project:restart-parent"); |
| 4760 | const runId = RunId.make("run:restart-parent"); |
| 4761 | const rootNodeId = NodeId.make("node:restart-parent-root"); |
| 4762 | yield* seedParentWithTerminalTask({ |
| 4763 | threadId, |
| 4764 | projectId, |
| 4765 | runId, |
| 4766 | rootNodeId, |
| 4767 | taskId: NodeId.make("node:restart-parent-settled"), |
| 4768 | deliveryState: "delivered", |
| 4769 | now, |
| 4770 | }); |
| 4771 | const child = ( |
| 4772 | name: string, |
| 4773 | continuationPending: boolean, |
| 4774 | runStatus?: "cancelled" | "completed", |
| 4775 | ) => |
| 4776 | seedRestartCancelledChild({ |
| 4777 | parentThreadId: threadId, |
| 4778 | projectId, |
| 4779 | parentRunId: runId, |
| 4780 | rootNodeId, |
| 4781 | name, |
| 4782 | completionWake: "always", |
| 4783 | continuationPending, |
| 4784 | ...(runStatus === undefined ? {} : { runStatus }), |
| 4785 | now, |
| 4786 | }); |
| 4787 | const resumed = yield* child("restart-resumed-child", true); |
| 4788 | const stopped = yield* child("restart-stopped-child", false); |
| 4789 | const backgrounded = yield* child("restart-backgrounded-child", true, "completed"); |
| 4790 | const recut = yield* child("restart-recut-child", false); |
| 4791 | const recutContinuationId = RunId.make("run:restart-recut-child:2"); |
| 4792 | const recutCommandId = CommandId.make("command:restart-recut-child:reconcile"); |
| 4793 | const recutRun = runEvent({ |
| 4794 | threadId: recut.childThreadId, |
| 4795 | runId: recutContinuationId, |
| 4796 | ordinal: 2, |
| 4797 | status: "cancelled", |
| 4798 | now, |
| 4799 | }); |
| 4800 | yield* eventSink.writeWithEffects({ |
| 4801 | commandId: recutCommandId, |
| 4802 | events: [ |
| 4803 | { |
| 4804 | ...recutRun, |
| 4805 | payload: { |
| 4806 | ...recutRun.payload, |
| 4807 | startedAt: null, |
| 4808 | restartContinuationOfRunId: recut.childRunId, |
| 4809 | }, |
| 4810 | }, |
| 4811 | ], |
| 4812 | effects: [ |
| 4813 | { |
| 4814 | id: `effect:restart-continuation:${recutContinuationId}`, |
| 4815 | commandId: recutCommandId, |
| 4816 | threadId: recut.childThreadId, |
| 4817 | request: { type: "provider-runtime.continue", sourceRunId: recutContinuationId }, |
| 4818 | }, |
| 4819 | ], |
| 4820 | }); |
| 4821 | |
| 4822 | yield* orchestrator.recoverDelegatedTasks; |
| 4823 | |
| 4824 | const recovered = yield* orchestrator.getThreadProjection(threadId); |
| 4825 | const task = (id: NodeId) => recovered.subagents.find((row) => row.id === id); |
| 4826 | assert.equal(task(stopped.taskId)?.status, "cancelled"); |
| 4827 | assert.equal(task(stopped.taskId)?.completionDelivery?.state, "claimed"); |
| 4828 | assert.equal(task(resumed.taskId)?.status, "running"); |
| 4829 | assert.isNull(task(resumed.taskId)?.result ?? null); |
| 4830 | assert.equal(task(backgrounded.taskId)?.status, "running"); |
| 4831 | assert.isNull(task(backgrounded.taskId)?.result ?? null); |
| 4832 | assert.equal(task(recut.taskId)?.status, "running"); |
| 4833 | assert.isTrue(yield* orchestrator.delegatedTaskResultPending(recut.childThreadId)); |
| 4834 | assert.isTrue(yield* orchestrator.delegatedTaskResultPending(resumed.childThreadId)); |
| 4835 | assert.isFalse(yield* orchestrator.delegatedTaskResultPending(stopped.childThreadId)); |
| 4836 | yield* orchestrator.recoverDelegatedTask(recut.childThreadId, { |
| 4837 | sourceRunId: recut.childRunId, |
| 4838 | }); |
| 4839 | const replayed = yield* orchestrator.getThreadProjection(threadId); |
| 4840 | assert.equal(replayed.subagents.find((row) => row.id === recut.taskId)?.status, "running"); |
| 4841 | yield* eventSink.write({ |
| 4842 | commandId: CommandId.make("command:restart-stopped-child:resumed"), |
| 4843 | events: [ |
| 4844 | runEvent({ |
| 4845 | threadId: stopped.childThreadId, |
| 4846 | runId: RunId.make("run:restart-stopped-child:2"), |
| 4847 | ordinal: 2, |
| 4848 | status: "running", |
| 4849 | now, |
| 4850 | }), |
| 4851 | ], |
| 4852 | }); |
| 4853 | assert.isTrue(yield* orchestrator.delegatedTaskResultPending(stopped.childThreadId)); |
| 4854 | assert.isFalse( |
| 4855 | recovered.contextTransfers.some( |
| 4856 | (transfer) => transfer.sourceThreadId === resumed.childThreadId, |
| 4857 | ), |
| 4858 | ); |
| 4859 | |
| 4860 | const afterSequence = yield* eventSink.latestSequence(); |
| 4861 | const continuationRunId = RunId.make("run:restart-resumed-child:2"); |
| 4862 | yield* eventSink.write({ |
| 4863 | commandId: CommandId.make("command:restart-resumed-child:completed"), |
| 4864 | events: [ |
| 4865 | { |
| 4866 | id: EventId.make("event:restart-resumed-child:result"), |
| 4867 | type: "message.updated", |
| 4868 | threadId: resumed.childThreadId, |
| 4869 | runId: continuationRunId, |
| 4870 | occurredAt: now, |
| 4871 | payload: { |
| 4872 | id: MessageId.make("message:restart-resumed-child:result"), |
| 4873 | threadId: resumed.childThreadId, |
| 4874 | runId: continuationRunId, |
| 4875 | nodeId: null, |
| 4876 | role: "assistant", |
| 4877 | text: "Finished after the restart.", |
| 4878 | attachments: [], |
| 4879 | streaming: false, |
| 4880 | createdBy: "agent", |
| 4881 | creationSource: "server", |
| 4882 | createdAt: now, |
| 4883 | updatedAt: now, |
| 4884 | }, |
| 4885 | }, |
| 4886 | runEvent({ |
| 4887 | threadId: resumed.childThreadId, |
| 4888 | runId: continuationRunId, |
| 4889 | ordinal: 2, |
| 4890 | status: "completed", |
| 4891 | now, |
| 4892 | }), |
| 4893 | ], |
| 4894 | }); |
| 4895 | const settled = yield* eventSink |
| 4896 | .stream({ afterSequence, eventType: "subagent.updated" }) |
| 4897 | .pipe( |
| 4898 | Stream.filter( |
| 4899 | (stored) => |
| 4900 | stored.event.type === "subagent.updated" && |
| 4901 | stored.event.payload.id === resumed.taskId, |
| 4902 | ), |
| 4903 | Stream.take(1), |
| 4904 | Stream.runHead, |
| 4905 | ); |
| 4906 | assert.isTrue(settled._tag === "Some"); |
| 4907 | const finished = yield* orchestrator.getThreadProjection(threadId); |
| 4908 | const finishedTask = finished.subagents.find((row) => row.id === resumed.taskId); |
| 4909 | assert.equal(finishedTask?.status, "completed"); |
| 4910 | assert.equal(finishedTask?.result, "Finished after the restart."); |
| 4911 | }), |
| 4912 | ); |
| 4913 | |
| 4914 | it.effect("settles a restart-cancelled child whose continuation declines to start", () => |
| 4915 | Effect.gen(function* () { |
| 4916 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 4917 | const now = yield* DateTime.now; |
| 4918 | const threadId = ThreadId.make("thread:restart-declined-parent"); |
| 4919 | const projectId = ProjectId.make("project:restart-declined-parent"); |
| 4920 | const runId = RunId.make("run:restart-declined-parent"); |
| 4921 | const rootNodeId = NodeId.make("node:restart-declined-parent-root"); |
| 4922 | yield* seedParentWithTerminalTask({ |
| 4923 | threadId, |
| 4924 | projectId, |
| 4925 | runId, |
| 4926 | rootNodeId, |
| 4927 | taskId: NodeId.make("node:restart-declined-parent-settled"), |
| 4928 | deliveryState: "delivered", |
| 4929 | now, |
| 4930 | }); |
| 4931 | const child = yield* seedRestartCancelledChild({ |
| 4932 | parentThreadId: threadId, |
| 4933 | projectId, |
| 4934 | parentRunId: runId, |
| 4935 | rootNodeId, |
| 4936 | name: "restart-declined-child", |
| 4937 | completionWake: "always", |
| 4938 | continuationPending: true, |
| 4939 | now, |
| 4940 | }); |
| 4941 | yield* orchestrator.recoverDelegatedTasks; |
| 4942 | const held = yield* orchestrator.getThreadProjection(threadId); |
| 4943 | assert.equal(held.subagents.find((row) => row.id === child.taskId)?.status, "running"); |
| 4944 | |
| 4945 | yield* continueRestartedRun({ |
| 4946 | threadId: child.childThreadId, |
| 4947 | sourceRunId: child.childRunId, |
| 4948 | }).pipe( |
| 4949 | Effect.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: false })), |
| 4950 | ); |
| 4951 | |
| 4952 | const settled = yield* orchestrator.getThreadProjection(threadId); |
| 4953 | const task = settled.subagents.find((row) => row.id === child.taskId); |
| 4954 | assert.equal(task?.status, "cancelled"); |
| 4955 | assert.equal(task?.completionDelivery?.state, "claimed"); |
| 4956 | |
| 4957 | // The parent's Stop committed, but the restart cut the effect that stops its children. |
| 4958 | const stoppedChild = yield* seedRestartCancelledChild({ |
| 4959 | parentThreadId: threadId, |
| 4960 | projectId, |
| 4961 | parentRunId: runId, |
| 4962 | rootNodeId, |
| 4963 | name: "restart-parent-stopped-child", |
| 4964 | completionWake: "always", |
| 4965 | continuationPending: true, |
| 4966 | now, |
| 4967 | }); |
| 4968 | yield* (yield* EventSink.EventSinkV2).write({ |
| 4969 | commandId: CommandId.make("command:restart-declined-parent:stop"), |
| 4970 | events: [ |
| 4971 | { |
| 4972 | id: EventId.make("event:restart-declined-parent:interrupt-request"), |
| 4973 | type: "turn-item.updated", |
| 4974 | threadId, |
| 4975 | runId, |
| 4976 | nodeId: rootNodeId, |
| 4977 | providerInstanceId: modelSelection.instanceId, |
| 4978 | occurredAt: now, |
| 4979 | payload: { |
| 4980 | id: TurnItemId.make("turn-item:restart-declined-parent:interrupt-request"), |
| 4981 | type: "run_interrupt_request", |
| 4982 | threadId, |
| 4983 | runId, |
| 4984 | nodeId: rootNodeId, |
| 4985 | providerThreadId: null, |
| 4986 | providerTurnId: null, |
| 4987 | nativeItemRef: null, |
| 4988 | parentItemId: null, |
| 4989 | ordinal: 90, |
| 4990 | status: "completed", |
| 4991 | title: "Interrupt requested", |
| 4992 | startedAt: now, |
| 4993 | completedAt: now, |
| 4994 | updatedAt: now, |
| 4995 | message: "Stopped by user", |
| 4996 | }, |
| 4997 | }, |
| 4998 | ], |
| 4999 | }); |
| 5000 | yield* continueRestartedRun({ |
Showing first 5,000 of 5,965 lines.
View raw