The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | import { assert, it, vi } from "@effect/vitest"; |
| 2 | import { |
| 3 | MessageId, |
| 4 | NodeId, |
| 5 | ProviderDriverKind, |
| 6 | ProviderInstanceId, |
| 7 | ProviderSessionId, |
| 8 | ProviderThreadId, |
| 9 | ProviderTurnId, |
| 10 | RunAttemptId, |
| 11 | RunId, |
| 12 | RuntimeRequestId, |
| 13 | ThreadId, |
| 14 | TurnItemId, |
| 15 | type OrchestrationV2ThreadProjection, |
| 16 | } from "@t3tools/contracts"; |
| 17 | import * as DateTime from "effect/DateTime"; |
| 18 | import * as Effect from "effect/Effect"; |
| 19 | import * as Layer from "effect/Layer"; |
| 20 | |
| 21 | import * as EffectWorker from "./EffectWorker.ts"; |
| 22 | import * as EffectOutbox from "./EffectOutbox.ts"; |
| 23 | import * as EventSink from "./EventSink.ts"; |
| 24 | import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; |
| 25 | import * as ProjectionStore from "./ProjectionStore.ts"; |
| 26 | import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts"; |
| 27 | import * as ServerSettings from "../serverSettings.ts"; |
| 28 | |
| 29 | it.effect("expires orphaned runtime requests before command readiness", () => { |
| 30 | const threadId = ThreadId.make("thread_recovery_requests"); |
| 31 | let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null = |
| 32 | null; |
| 33 | const committed = vi.fn( |
| 34 | (input: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0]) => { |
| 35 | committedInput = input; |
| 36 | return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); |
| 37 | }, |
| 38 | ); |
| 39 | const projection = { |
| 40 | thread: { id: threadId }, |
| 41 | runtimeRequests: [ |
| 42 | { |
| 43 | id: RuntimeRequestId.make("request_orphaned"), |
| 44 | nodeId: NodeId.make("node_orphaned"), |
| 45 | status: "pending", |
| 46 | responseCapability: { type: "not_resumable", reason: "old process" }, |
| 47 | }, |
| 48 | ], |
| 49 | providerSessions: [], |
| 50 | providerThreads: [], |
| 51 | runs: [], |
| 52 | nodes: [], |
| 53 | } as unknown as OrchestrationV2ThreadProjection; |
| 54 | const layer = ProviderRuntimeRecovery.layer.pipe( |
| 55 | Layer.provide(ServerSettings.layerTest()), |
| 56 | Layer.provide( |
| 57 | Layer.mergeAll( |
| 58 | Layer.mock(ProjectionStore.ProjectionStoreV2)({ |
| 59 | getRecoveryThreadIds: () => Effect.succeed([threadId]), |
| 60 | getRuntimeRecoveryProjection: () => Effect.succeed(projection), |
| 61 | }), |
| 62 | Layer.mock(EventSink.EventSinkV2)({ commitCommand: committed }), |
| 63 | IdAllocator.layer, |
| 64 | Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ |
| 65 | runRecoveryOnce: Effect.succeed(false), |
| 66 | }), |
| 67 | Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 68 | reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), |
| 69 | }), |
| 70 | ), |
| 71 | ), |
| 72 | ); |
| 73 | return Effect.gen(function* () { |
| 74 | yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).recover; |
| 75 | const command = committedInput; |
| 76 | assert.isNotNull(command); |
| 77 | if (command === null) return; |
| 78 | assert.equal(command?.events[0]?.type, "runtime-request.updated"); |
| 79 | if (command?.events[0]?.type === "runtime-request.updated") { |
| 80 | assert.equal(command.events[0].payload.status, "expired"); |
| 81 | assert.equal(command.events[0].payload.responseCapability.type, "not_resumable"); |
| 82 | } |
| 83 | }).pipe(Effect.provide(layer)); |
| 84 | }); |
| 85 | |
| 86 | it.effect("preserves async questions across startup and shutdown", () => { |
| 87 | const threadId = ThreadId.make("async-recovery-thread"); |
| 88 | const nodeId = NodeId.make("async-recovery-node"); |
| 89 | const projection = { |
| 90 | thread: { id: threadId }, |
| 91 | runtimeRequests: [ |
| 92 | { |
| 93 | id: RuntimeRequestId.make("async-recovery-request"), |
| 94 | nodeId, |
| 95 | status: "pending", |
| 96 | responseCapability: { type: "message" }, |
| 97 | }, |
| 98 | ], |
| 99 | providerSessions: [], |
| 100 | providerThreads: [], |
| 101 | runs: [], |
| 102 | nodes: [], |
| 103 | turnItems: [], |
| 104 | } as unknown as OrchestrationV2ThreadProjection; |
| 105 | const commitCommand = vi.fn(() => Effect.die("an async question needs no process-loss write")); |
| 106 | const layer = ProviderRuntimeRecovery.layer.pipe( |
| 107 | Layer.provide(ServerSettings.layerTest()), |
| 108 | Layer.provide( |
| 109 | Layer.mergeAll( |
| 110 | Layer.mock(ProjectionStore.ProjectionStoreV2)({ |
| 111 | getRecoveryThreadIds: () => Effect.succeed([threadId]), |
| 112 | getRuntimeRecoveryProjection: () => Effect.succeed(projection), |
| 113 | }), |
| 114 | Layer.mock(EventSink.EventSinkV2)({ commitCommand }), |
| 115 | IdAllocator.layer, |
| 116 | Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ |
| 117 | runRecoveryOnce: Effect.succeed(false), |
| 118 | }), |
| 119 | Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 120 | reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), |
| 121 | cancelUnsettled: () => Effect.succeed([]), |
| 122 | signalCancellations: () => Effect.void, |
| 123 | }), |
| 124 | ), |
| 125 | ), |
| 126 | ); |
| 127 | return Effect.gen(function* () { |
| 128 | const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService; |
| 129 | assert.equal((yield* recovery.reconcile("startup")).closedRequests, 0); |
| 130 | assert.equal((yield* recovery.reconcile("startup")).closedRequests, 0); |
| 131 | assert.isFalse(commitCommand.mock.calls.length > 0); |
| 132 | }).pipe(Effect.provide(layer)); |
| 133 | }); |
| 134 | |
| 135 | it.effect( |
| 136 | "preserves a replayable waiting run while cancelling its process-bound background work", |
| 137 | () => { |
| 138 | const threadId = ThreadId.make("thread_waiting_checkpoint"); |
| 139 | const runId = RunId.make("run_waiting_checkpoint"); |
| 140 | const providerThreadId = ProviderThreadId.make("provider_thread_waiting_checkpoint"); |
| 141 | const providerInstanceId = ProviderInstanceId.make("claude"); |
| 142 | const backgroundItemId = TurnItemId.make("turn_item_waiting_checkpoint"); |
| 143 | let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null = |
| 144 | null; |
| 145 | const projection = { |
| 146 | thread: { id: threadId }, |
| 147 | runtimeRequests: [], |
| 148 | providerSessions: [], |
| 149 | providerThreads: [ |
| 150 | { |
| 151 | id: providerThreadId, |
| 152 | driver: ProviderDriverKind.make("claude"), |
| 153 | providerInstanceId, |
| 154 | status: "idle", |
| 155 | pendingBackgroundTasks: [ |
| 156 | { taskId: String(backgroundItemId), description: "Finish background task" }, |
| 157 | ], |
| 158 | }, |
| 159 | ], |
| 160 | runs: [{ id: runId, status: "waiting", providerInstanceId, providerThreadId }], |
| 161 | attempts: [], |
| 162 | nodes: [], |
| 163 | subagents: [], |
| 164 | messages: [], |
| 165 | turnItems: [ |
| 166 | { |
| 167 | id: backgroundItemId, |
| 168 | runId, |
| 169 | nodeId: null, |
| 170 | providerThreadId, |
| 171 | type: "command_execution", |
| 172 | status: "running", |
| 173 | }, |
| 174 | ], |
| 175 | } as unknown as OrchestrationV2ThreadProjection; |
| 176 | const layer = ProviderRuntimeRecovery.layer.pipe( |
| 177 | Layer.provide(ServerSettings.layerTest()), |
| 178 | Layer.provide( |
| 179 | Layer.mergeAll( |
| 180 | Layer.mock(ProjectionStore.ProjectionStoreV2)({ |
| 181 | getRecoveryThreadIds: () => Effect.succeed([threadId]), |
| 182 | getRuntimeRecoveryProjection: () => Effect.succeed(projection), |
| 183 | }), |
| 184 | Layer.mock(EventSink.EventSinkV2)({ |
| 185 | commitCommand: (input) => { |
| 186 | committedInput = input; |
| 187 | return Effect.succeed({ |
| 188 | committed: true, |
| 189 | cancelledEffectCount: 0, |
| 190 | } as never); |
| 191 | }, |
| 192 | }), |
| 193 | IdAllocator.layer, |
| 194 | Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ |
| 195 | runRecoveryOnce: Effect.succeed(false), |
| 196 | }), |
| 197 | Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 198 | listByCommandId: () => |
| 199 | Effect.succeed([ |
| 200 | { |
| 201 | request: { type: "checkpoint.capture", runId }, |
| 202 | status: "running", |
| 203 | }, |
| 204 | ] as never), |
| 205 | reconcileAfterProcessLoss: Effect.succeed({ requeued: 1, cancelled: 0 }), |
| 206 | }), |
| 207 | ), |
| 208 | ), |
| 209 | ); |
| 210 | |
| 211 | return Effect.gen(function* () { |
| 212 | const summary = |
| 213 | yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); |
| 214 | assert.equal(summary.terminalizedRuns, 0); |
| 215 | assert.equal(summary.requeuedEffects, 1); |
| 216 | |
| 217 | const command = committedInput; |
| 218 | assert.isNotNull(command); |
| 219 | if (command === null) return; |
| 220 | |
| 221 | assert.isFalse(command.events.some((event) => event.type === "run.updated")); |
| 222 | const recorded = command.events.flatMap((event) => |
| 223 | event.type === "run.background-work-cancelled" ? [event.payload] : [], |
| 224 | ); |
| 225 | |
| 226 | assert.deepEqual( |
| 227 | recorded.map((entry) => ({ |
| 228 | runId: entry.runId, |
| 229 | kinds: entry.restartCancelledBackgroundWork.map((work) => work.kind), |
| 230 | })), |
| 231 | [{ runId, kinds: ["shell"] }], |
| 232 | ); |
| 233 | assert.isTrue( |
| 234 | command.events.some( |
| 235 | (event) => |
| 236 | event.type === "turn-item.updated" && |
| 237 | event.payload.id === backgroundItemId && |
| 238 | event.payload.status === "cancelled", |
| 239 | ), |
| 240 | ); |
| 241 | assert.isTrue( |
| 242 | command.events.some( |
| 243 | (event) => |
| 244 | event.type === "provider-thread.updated" && |
| 245 | event.payload.id === providerThreadId && |
| 246 | event.payload.pendingBackgroundTasks?.length === 0, |
| 247 | ), |
| 248 | ); |
| 249 | }).pipe(Effect.provide(layer)); |
| 250 | }, |
| 251 | ); |
| 252 | |
| 253 | it.effect( |
| 254 | "records cancelled work on the provider thread that lost it after a provider switch", |
| 255 | () => { |
| 256 | const threadId = ThreadId.make("thread_switch_note"); |
| 257 | const claudeRunId = RunId.make("run_switch_note_claude"); |
| 258 | const codexRunId = RunId.make("run_switch_note_codex"); |
| 259 | const claudeThreadId = ProviderThreadId.make("provider_thread_switch_note_claude"); |
| 260 | const codexThreadId = ProviderThreadId.make("provider_thread_switch_note_codex"); |
| 261 | const claude = ProviderInstanceId.make("claude"); |
| 262 | const codex = ProviderInstanceId.make("codex"); |
| 263 | let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null = |
| 264 | null; |
| 265 | |
| 266 | const projection = { |
| 267 | thread: { id: threadId }, |
| 268 | runtimeRequests: [], |
| 269 | providerSessions: [], |
| 270 | providerThreads: [ |
| 271 | { |
| 272 | id: claudeThreadId, |
| 273 | ownerNodeId: null, |
| 274 | driver: ProviderDriverKind.make("claudeAgent"), |
| 275 | providerInstanceId: claude, |
| 276 | status: "idle", |
| 277 | pendingBackgroundTasks: [{ taskId: "task-claude", description: "Watch the build" }], |
| 278 | }, |
| 279 | { |
| 280 | id: codexThreadId, |
| 281 | ownerNodeId: null, |
| 282 | driver: ProviderDriverKind.make("codex"), |
| 283 | providerInstanceId: codex, |
| 284 | status: "idle", |
| 285 | }, |
| 286 | ], |
| 287 | providerTurns: [], |
| 288 | runs: [ |
| 289 | { |
| 290 | id: claudeRunId, |
| 291 | ordinal: 1, |
| 292 | status: "completed", |
| 293 | providerInstanceId: claude, |
| 294 | providerThreadId: claudeThreadId, |
| 295 | }, |
| 296 | { |
| 297 | id: codexRunId, |
| 298 | ordinal: 2, |
| 299 | status: "completed", |
| 300 | providerInstanceId: codex, |
| 301 | providerThreadId: codexThreadId, |
| 302 | }, |
| 303 | ], |
| 304 | attempts: [], |
| 305 | nodes: [], |
| 306 | subagents: [], |
| 307 | messages: [], |
| 308 | turnItems: [ |
| 309 | { |
| 310 | id: TurnItemId.make("turn_item_switch_note_subagent"), |
| 311 | runId: claudeRunId, |
| 312 | nodeId: null, |
| 313 | providerThreadId: claudeThreadId, |
| 314 | nativeItemRef: null, |
| 315 | type: "subagent", |
| 316 | title: "Background subagent", |
| 317 | status: "running", |
| 318 | }, |
| 319 | ], |
| 320 | } as unknown as OrchestrationV2ThreadProjection; |
| 321 | const layer = ProviderRuntimeRecovery.layer.pipe( |
| 322 | Layer.provide(ServerSettings.layerTest()), |
| 323 | Layer.provide( |
| 324 | Layer.mergeAll( |
| 325 | Layer.mock(ProjectionStore.ProjectionStoreV2)({ |
| 326 | getRecoveryThreadIds: () => Effect.succeed([threadId]), |
| 327 | getRuntimeRecoveryProjection: () => Effect.succeed(projection), |
| 328 | }), |
| 329 | Layer.mock(EventSink.EventSinkV2)({ |
| 330 | commitCommand: (input) => { |
| 331 | committedInput = input; |
| 332 | return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); |
| 333 | }, |
| 334 | }), |
| 335 | IdAllocator.layer, |
| 336 | Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ |
| 337 | runRecoveryOnce: Effect.succeed(false), |
| 338 | }), |
| 339 | Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 340 | reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), |
| 341 | }), |
| 342 | ), |
| 343 | ), |
| 344 | ); |
| 345 | |
| 346 | return Effect.gen(function* () { |
| 347 | yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); |
| 348 | const recorded = (committedInput?.events ?? []).flatMap((event) => |
| 349 | event.type === "run.background-work-cancelled" ? [event.payload] : [], |
| 350 | ); |
| 351 | |
| 352 | assert.deepEqual( |
| 353 | recorded.map((entry) => ({ |
| 354 | runId: entry.runId, |
| 355 | kinds: entry.restartCancelledBackgroundWork.map((work) => work.kind), |
| 356 | })), |
| 357 | [{ runId: claudeRunId, kinds: ["subagent", "task"] }], |
| 358 | ); |
| 359 | }).pipe(Effect.provide(layer)); |
| 360 | }, |
| 361 | ); |
| 362 | |
| 363 | it.effect("cancels a stale waiting run when no checkpoint capture can finish it", () => { |
| 364 | const threadId = ThreadId.make("thread_stale_waiting"); |
| 365 | const runId = RunId.make("run_stale_waiting"); |
| 366 | let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null = |
| 367 | null; |
| 368 | const projection = { |
| 369 | thread: { id: threadId }, |
| 370 | runtimeRequests: [], |
| 371 | providerSessions: [], |
| 372 | providerThreads: [], |
| 373 | providerTurns: [], |
| 374 | runs: [ |
| 375 | { |
| 376 | id: runId, |
| 377 | status: "waiting", |
| 378 | providerInstanceId: ProviderInstanceId.make("codex"), |
| 379 | }, |
| 380 | ], |
| 381 | attempts: [], |
| 382 | nodes: [], |
| 383 | subagents: [], |
| 384 | messages: [], |
| 385 | turnItems: [], |
| 386 | } as unknown as OrchestrationV2ThreadProjection; |
| 387 | const layer = ProviderRuntimeRecovery.layer.pipe( |
| 388 | Layer.provide(ServerSettings.layerTest()), |
| 389 | Layer.provide( |
| 390 | Layer.mergeAll( |
| 391 | Layer.mock(ProjectionStore.ProjectionStoreV2)({ |
| 392 | getRecoveryThreadIds: () => Effect.succeed([threadId]), |
| 393 | getRuntimeRecoveryProjection: () => Effect.succeed(projection), |
| 394 | }), |
| 395 | Layer.mock(EventSink.EventSinkV2)({ |
| 396 | commitCommand: (input) => { |
| 397 | committedInput = input; |
| 398 | return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); |
| 399 | }, |
| 400 | }), |
| 401 | IdAllocator.layer, |
| 402 | Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ |
| 403 | runRecoveryOnce: Effect.succeed(false), |
| 404 | }), |
| 405 | Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 406 | listByCommandId: () => Effect.succeed([]), |
| 407 | reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), |
| 408 | }), |
| 409 | ), |
| 410 | ), |
| 411 | ); |
| 412 | |
| 413 | return Effect.gen(function* () { |
| 414 | const summary = |
| 415 | yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); |
| 416 | assert.equal(summary.terminalizedRuns, 1); |
| 417 | const runEvent = committedInput?.events.find((event) => event.type === "run.updated"); |
| 418 | assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.status : null, "cancelled"); |
| 419 | }).pipe(Effect.provide(layer)); |
| 420 | }); |
| 421 | |
| 422 | it.effect("recovery closes a secret card but leaves a link card and an unread steer open", () => { |
| 423 | const threadId = ThreadId.make("thread_secret_recovery"); |
| 424 | const runId = RunId.make("run_secret_recovery"); |
| 425 | let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null = |
| 426 | null; |
| 427 | const projection = { |
| 428 | thread: { id: threadId }, |
| 429 | runtimeRequests: [], |
| 430 | providerSessions: [], |
| 431 | providerThreads: [], |
| 432 | providerTurns: [ |
| 433 | { runAttemptId: RunAttemptId.make("attempt_secret_recovery"), status: "running" }, |
| 434 | ], |
| 435 | runs: [ |
| 436 | { |
| 437 | id: runId, |
| 438 | status: "running", |
| 439 | activeAttemptId: RunAttemptId.make("attempt_secret_recovery"), |
| 440 | providerInstanceId: ProviderInstanceId.make("codex"), |
| 441 | }, |
| 442 | ], |
| 443 | attempts: [], |
| 444 | nodes: [], |
| 445 | subagents: [], |
| 446 | messages: [], |
| 447 | turnItems: [ |
| 448 | { |
| 449 | id: "turn-item:secret-request:recovery", |
| 450 | threadId, |
| 451 | runId, |
| 452 | nodeId: null, |
| 453 | type: "secret_request", |
| 454 | status: "waiting", |
| 455 | secretStatus: "pending", |
| 456 | label: "GitHub token", |
| 457 | reason: "Used as GH_TOKEN.", |
| 458 | }, |
| 459 | { |
| 460 | id: "turn-item:link-request:recovery", |
| 461 | threadId, |
| 462 | runId, |
| 463 | nodeId: null, |
| 464 | type: "link_request", |
| 465 | status: "waiting", |
| 466 | linkStatus: "pending", |
| 467 | url: "https://box.example.ts.net", |
| 468 | reason: "Run the suite there.", |
| 469 | }, |
| 470 | { |
| 471 | id: "turn-item:steer:recovery", |
| 472 | threadId, |
| 473 | runId, |
| 474 | nodeId: null, |
| 475 | type: "user_message", |
| 476 | status: "pending", |
| 477 | inputIntent: "steer", |
| 478 | messageId: "message:steer:recovery", |
| 479 | }, |
| 480 | ], |
| 481 | } as unknown as OrchestrationV2ThreadProjection; |
| 482 | const layer = ProviderRuntimeRecovery.layer.pipe( |
| 483 | Layer.provide(ServerSettings.layerTest()), |
| 484 | Layer.provide( |
| 485 | Layer.mergeAll( |
| 486 | Layer.mock(ProjectionStore.ProjectionStoreV2)({ |
| 487 | getRecoveryThreadIds: () => Effect.succeed([threadId]), |
| 488 | getRuntimeRecoveryProjection: () => Effect.succeed(projection), |
| 489 | }), |
| 490 | Layer.mock(EventSink.EventSinkV2)({ |
| 491 | commitCommand: (input) => { |
| 492 | committedInput = input; |
| 493 | return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); |
| 494 | }, |
| 495 | }), |
| 496 | IdAllocator.layer, |
| 497 | Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ |
| 498 | runRecoveryOnce: Effect.succeed(false), |
| 499 | }), |
| 500 | Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 501 | listByCommandId: () => Effect.succeed([]), |
| 502 | reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), |
| 503 | }), |
| 504 | ), |
| 505 | ), |
| 506 | ); |
| 507 | |
| 508 | return Effect.gen(function* () { |
| 509 | yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); |
| 510 | const cards = (committedInput?.events ?? []).flatMap((event) => |
| 511 | event.type === "turn-item.updated" ? [event.payload] : [], |
| 512 | ); |
| 513 | const secret = cards.find((card) => card.type === "secret_request"); |
| 514 | assert.equal(secret?.status, "cancelled"); |
| 515 | assert.equal(secret?.type === "secret_request" ? secret.secretStatus : null, "cancelled"); |
| 516 | // Nothing in memory waits on a link card; the user can still answer it. |
| 517 | // Claude never reported reading the steer, so the orchestrator queues it. |
| 518 | assert.deepEqual( |
| 519 | cards.filter((card) => card.type === "link_request" || card.type === "user_message"), |
| 520 | [], |
| 521 | ); |
| 522 | }).pipe(Effect.provide(layer)); |
| 523 | }); |
| 524 | |
| 525 | it.effect.each([ |
| 526 | "queued", |
| 527 | "preparing", |
| 528 | "starting", |
| 529 | "starting-other-request", |
| 530 | "starting-own-request", |
| 531 | "starting-scheduled", |
| 532 | "starting-steered", |
| 533 | "running", |
| 534 | "running-delegated", |
| 535 | "running-delegated-capped", |
| 536 | "running-delegated-snoozed", |
| 537 | "running-delegated-stopped", |
| 538 | "running-resume", |
| 539 | ] as const)("recovers $0 work after restart", (scenario) => { |
| 540 | const status = scenario.startsWith("starting") |
| 541 | ? "starting" |
| 542 | : scenario.startsWith("running") |
| 543 | ? "running" |
| 544 | : scenario; |
| 545 | const resent = scenario === "starting-scheduled" || scenario === "running-delegated"; |
| 546 | const terminalized = |
| 547 | status === "preparing" || |
| 548 | scenario === "starting-own-request" || |
| 549 | scenario === "starting-steered" || |
| 550 | scenario === "running-delegated-capped" || |
| 551 | scenario === "running-delegated-snoozed" || |
| 552 | scenario === "running-delegated-stopped" || |
| 553 | scenario === "running-resume"; |
| 554 | const startedAt = |
| 555 | status === "running" || scenario === "starting-steered" |
| 556 | ? DateTime.makeUnsafe("2026-11-01T19:34:40.403Z") |
| 557 | : null; |
| 558 | const threadId = ThreadId.make("thread_queued_restart"); |
| 559 | const runId = RunId.make("run_queued_restart"); |
| 560 | const attemptId = RunAttemptId.make("attempt_queued_restart"); |
| 561 | const steeredAttemptId = RunAttemptId.make("attempt_queued_restart_steered"); |
| 562 | const rootNodeId = NodeId.make("node_queued_restart"); |
| 563 | let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null = |
| 564 | null; |
| 565 | const written: Array<EffectOutbox.PendingOrchestrationEffectV2> = []; |
| 566 | const projection = { |
| 567 | thread: { |
| 568 | id: threadId, |
| 569 | archivedAt: null, |
| 570 | deletedAt: null, |
| 571 | settledOverride: null, |
| 572 | snoozedUntil: |
| 573 | scenario === "running-delegated-snoozed" |
| 574 | ? DateTime.makeUnsafe("2099-07-25T09:00:00.000Z") |
| 575 | : null, |
| 576 | lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, |
| 577 | forkedFrom: null, |
| 578 | ...(scenario === "running-resume" |
| 579 | ? { |
| 580 | limitRecovery: { |
| 581 | runId: RunId.make("run_queued_restart_limited"), |
| 582 | resetAt: "2026-11-01T19:34:00.000Z", |
| 583 | autoResume: true, |
| 584 | }, |
| 585 | } |
| 586 | : {}), |
| 587 | }, |
| 588 | runtimeRequests: scenario.endsWith("request") |
| 589 | ? [ |
| 590 | { |
| 591 | id: RuntimeRequestId.make("pending-recovery-request"), |
| 592 | nodeId: |
| 593 | scenario === "starting-own-request" ? rootNodeId : NodeId.make("other-run-request"), |
| 594 | kind: "user_input", |
| 595 | status: "pending", |
| 596 | responseCapability: { type: "message" }, |
| 597 | }, |
| 598 | ] |
| 599 | : [], |
| 600 | providerSessions: [], |
| 601 | providerThreads: [], |
| 602 | providerTurns: |
| 603 | scenario === "starting-steered" |
| 604 | ? [ |
| 605 | { |
| 606 | id: ProviderTurnId.make("provider_turn_queued_restart_steered"), |
| 607 | runAttemptId: steeredAttemptId, |
| 608 | status: "interrupted", |
| 609 | }, |
| 610 | ] |
| 611 | : [], |
| 612 | runs: [ |
| 613 | { |
| 614 | id: runId, |
| 615 | status, |
| 616 | startedAt, |
| 617 | rootNodeId, |
| 618 | activeAttemptId: attemptId, |
| 619 | userMessageId: MessageId.make("message_queued_restart"), |
| 620 | queuePosition: 1, |
| 621 | providerInstanceId: ProviderInstanceId.make("codex"), |
| 622 | ...(scenario === "running-resume" |
| 623 | ? { usageLimitContinuationOfRunId: RunId.make("run_queued_restart_limited") } |
| 624 | : {}), |
| 625 | ...(status === "preparing" |
| 626 | ? { |
| 627 | workspacePreparation: { |
| 628 | type: "worktree", |
| 629 | baseRef: "parent-local-commit", |
| 630 | startFromOrigin: false, |
| 631 | }, |
| 632 | } |
| 633 | : {}), |
| 634 | }, |
| 635 | ], |
| 636 | attempts: [ |
| 637 | ...(scenario === "starting-steered" |
| 638 | ? [{ id: steeredAttemptId, runId, rootNodeId, status: "interrupted" }] |
| 639 | : []), |
| 640 | { |
| 641 | id: attemptId, |
| 642 | runId, |
| 643 | rootNodeId, |
| 644 | status: status === "running" ? "running" : "pending", |
| 645 | startedAt: status === "running" ? startedAt : null, |
| 646 | }, |
| 647 | ], |
| 648 | nodes: [ |
| 649 | ...(scenario === "starting-other-request" |
| 650 | ? [ |
| 651 | { |
| 652 | id: NodeId.make("other-run-request"), |
| 653 | runId: RunId.make("earlier-run"), |
| 654 | status: "waiting", |
| 655 | }, |
| 656 | ] |
| 657 | : []), |
| 658 | { |
| 659 | id: rootNodeId, |
| 660 | runId, |
| 661 | status: status === "running" ? "running" : "pending", |
| 662 | startedAt: status === "running" ? startedAt : null, |
| 663 | }, |
| 664 | ], |
| 665 | subagents: [], |
| 666 | messages: |
| 667 | status === "starting" || status === "running" |
| 668 | ? [ |
| 669 | { |
| 670 | id: MessageId.make("message_queued_restart"), |
| 671 | ...(scenario.startsWith("running-delegated") |
| 672 | ? { createdBy: "agent", creationSource: "mcp" } |
| 673 | : scenario === "running-resume" |
| 674 | ? { createdBy: "agent", creationSource: "server" } |
| 675 | : { createdBy: "user" }), |
| 676 | ...(scenario === "starting-scheduled" ? { scheduledTaskId: "scheduled-task" } : {}), |
| 677 | }, |
| 678 | ] |
| 679 | : [], |
| 680 | turnItems: |
| 681 | scenario === "running-delegated-stopped" |
| 682 | ? [ |
| 683 | { |
| 684 | id: TurnItemId.make("turn_item_queued_restart_stop"), |
| 685 | type: "run_interrupt_request", |
| 686 | runId, |
| 687 | nodeId: rootNodeId, |
| 688 | status: "completed", |
| 689 | }, |
| 690 | ] |
| 691 | : [], |
| 692 | } as unknown as OrchestrationV2ThreadProjection; |
| 693 | const layer = ProviderRuntimeRecovery.layer.pipe( |
| 694 | Layer.provide(ServerSettings.layerTest()), |
| 695 | Layer.provide( |
| 696 | Layer.mergeAll( |
| 697 | Layer.mock(ProjectionStore.ProjectionStoreV2)({ |
| 698 | getRecoveryThreadIds: () => Effect.succeed([threadId]), |
| 699 | // Like the store, recovery state carries only provider turns that have not ended. |
| 700 | getRuntimeRecoveryProjection: () => |
| 701 | Effect.succeed({ |
| 702 | ...projection, |
| 703 | providerTurns: projection.providerTurns.filter((turn) => turn.status === "running"), |
| 704 | }), |
| 705 | getMessagesById: (requestedThreadId, messageIds) => |
| 706 | Effect.succeed( |
| 707 | new Map( |
| 708 | projection.messages |
| 709 | .filter( |
| 710 | (message) => requestedThreadId === threadId && messageIds.includes(message.id), |
| 711 | ) |
| 712 | .map((message) => [message.id, message]), |
| 713 | ), |
| 714 | ), |
| 715 | getThreadRecords: () => Effect.succeed(projection), |
| 716 | }), |
| 717 | Layer.mock(EventSink.EventSinkV2)({ |
| 718 | commitCommand: (input) => { |
| 719 | committedInput = input; |
| 720 | return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); |
| 721 | }, |
| 722 | writeWithEffects: (input) => { |
| 723 | written.push(...input.effects); |
| 724 | return Effect.succeed([]); |
| 725 | }, |
| 726 | }), |
| 727 | IdAllocator.layer, |
| 728 | Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ |
| 729 | runRecoveryOnce: Effect.succeed(false), |
| 730 | }), |
| 731 | Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 732 | // Three earlier restarts each cut a resend the worker had claimed. |
| 733 | listByCommandId: () => |
| 734 | Effect.succeed( |
| 735 | scenario === "running-delegated-capped" |
| 736 | ? ([1, 1, 1].map((attemptCount) => ({ attemptCount })) as never) |
| 737 | : [], |
| 738 | ), |
| 739 | reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), |
| 740 | }), |
| 741 | ), |
| 742 | ), |
| 743 | ); |
| 744 | |
| 745 | return Effect.gen(function* () { |
| 746 | const summary = |
| 747 | yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); |
| 748 | assert.equal(summary.terminalizedRuns, terminalized ? 1 : 0); |
| 749 | const command = committedInput; |
| 750 | assert.isNotNull(command); |
| 751 | if (command === null) return; |
| 752 | const runEvent = command.events.find((event) => event.type === "run.updated"); |
| 753 | const attemptEvent = command.events.find((event) => event.type === "run-attempt.updated"); |
| 754 | const nodeEvent = command.events.find((event) => event.type === "node.updated"); |
| 755 | assert.deepInclude(runEvent?.type === "run.updated" ? runEvent.payload : null, { |
| 756 | status: terminalized ? "cancelled" : resent ? "starting" : "queued", |
| 757 | startedAt: scenario === "starting-steered" ? startedAt : null, |
| 758 | }); |
| 759 | assert.isEmpty(command.effects); |
| 760 | assert.deepEqual( |
| 761 | written.map((effect) => effect.request), |
| 762 | resent ? [{ type: "provider-turn.start", runId }] : [], |
| 763 | ); |
| 764 | if (terminalized) { |
| 765 | assert.equal( |
| 766 | attemptEvent?.type === "run-attempt.updated" ? attemptEvent.payload.status : null, |
| 767 | "cancelled", |
| 768 | ); |
| 769 | assert.equal( |
| 770 | nodeEvent?.type === "node.updated" ? nodeEvent.payload.status : null, |
| 771 | scenario === "starting-own-request" ? null : "cancelled", |
| 772 | ); |
| 773 | } else { |
| 774 | if (!resent) { |
| 775 | assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.queuePosition : null, 1); |
| 776 | assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.queueHeld : false, true); |
| 777 | } |
| 778 | if (status === "running") { |
| 779 | assert.deepInclude( |
| 780 | attemptEvent?.type === "run-attempt.updated" ? attemptEvent.payload : null, |
| 781 | { status: "pending", startedAt: null }, |
| 782 | ); |
| 783 | assert.deepInclude(nodeEvent?.type === "node.updated" ? nodeEvent.payload : null, { |
| 784 | status: "pending", |
| 785 | startedAt: null, |
| 786 | }); |
| 787 | } else { |
| 788 | assert.isUndefined(attemptEvent); |
| 789 | assert.isUndefined(nodeEvent); |
| 790 | } |
| 791 | } |
| 792 | // The unsent auto-resume gets a fresh request id so the limit worker sends it again. |
| 793 | if (scenario === "running-resume") { |
| 794 | const rearmed = command.events.find((event) => event.type === "thread.metadata-updated"); |
| 795 | assert.equal( |
| 796 | rearmed?.type === "thread.metadata-updated" |
| 797 | ? rearmed.payload.limitRecovery?.requestId |
| 798 | : null, |
| 799 | command.commandId, |
| 800 | ); |
| 801 | } |
| 802 | }).pipe(Effect.provide(layer)); |
| 803 | }); |
| 804 | |
| 805 | it.effect( |
| 806 | "cancels the complete in-flight subtree and stops its persisted session without reopening it", |
| 807 | () => { |
| 808 | const threadId = ThreadId.make("thread_recovery_cancel"); |
| 809 | const runId = RunId.make("run_recovery_cancel"); |
| 810 | const attemptId = RunAttemptId.make("attempt_recovery_cancel"); |
| 811 | const rootNodeId = NodeId.make("node_recovery_cancel"); |
| 812 | const providerThreadId = ProviderThreadId.make("provider_thread_recovery_cancel"); |
| 813 | const providerTurnId = ProviderTurnId.make("provider_turn_recovery_cancel"); |
| 814 | const providerSessionId = ProviderSessionId.make("provider_session_recovery_cancel"); |
| 815 | let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null = |
| 816 | null; |
| 817 | const projection = { |
| 818 | thread: { id: threadId }, |
| 819 | runtimeRequests: [], |
| 820 | providerSessions: [ |
| 821 | { |
| 822 | id: providerSessionId, |
| 823 | driver: ProviderDriverKind.make("codex"), |
| 824 | providerInstanceId: ProviderInstanceId.make("codex"), |
| 825 | status: "ready", |
| 826 | extensionStatuses: [{ key: "watch", text: "Watching" }], |
| 827 | }, |
| 828 | ], |
| 829 | providerThreads: [ |
| 830 | { |
| 831 | id: providerThreadId, |
| 832 | driver: ProviderDriverKind.make("codex"), |
| 833 | providerInstanceId: ProviderInstanceId.make("codex"), |
| 834 | status: "active", |
| 835 | }, |
| 836 | ], |
| 837 | providerTurns: [ |
| 838 | { |
| 839 | id: providerTurnId, |
| 840 | runAttemptId: attemptId, |
| 841 | nodeId: rootNodeId, |
| 842 | status: "running", |
| 843 | }, |
| 844 | ], |
| 845 | runs: [ |
| 846 | { |
| 847 | id: runId, |
| 848 | status: "starting", |
| 849 | activeAttemptId: attemptId, |
| 850 | providerThreadId, |
| 851 | providerInstanceId: ProviderInstanceId.make("codex"), |
| 852 | }, |
| 853 | ], |
| 854 | attempts: [ |
| 855 | { |
| 856 | id: attemptId, |
| 857 | runId, |
| 858 | rootNodeId, |
| 859 | status: "running", |
| 860 | }, |
| 861 | ], |
| 862 | nodes: [{ id: rootNodeId, runId, status: "running" }], |
| 863 | subagents: [], |
| 864 | messages: [{ id: MessageId.make("message_recovery_cancel"), runId, streaming: true }], |
| 865 | turnItems: [ |
| 866 | { |
| 867 | id: TurnItemId.make("turn_item_recovery_cancel"), |
| 868 | runId, |
| 869 | nodeId: rootNodeId, |
| 870 | status: "running", |
| 871 | }, |
| 872 | ...(["assistant_message", "reasoning", "proposed_plan"] as const).map((type) => ({ |
| 873 | id: TurnItemId.make(`turn_item_recovery_cancel_${type}`), |
| 874 | runId, |
| 875 | nodeId: rootNodeId, |
| 876 | type, |
| 877 | status: "running", |
| 878 | streaming: true, |
| 879 | })), |
| 880 | ], |
| 881 | } as unknown as OrchestrationV2ThreadProjection; |
| 882 | const layer = ProviderRuntimeRecovery.layer.pipe( |
| 883 | Layer.provide(ServerSettings.layerTest()), |
| 884 | Layer.provide( |
| 885 | Layer.mergeAll( |
| 886 | Layer.mock(ProjectionStore.ProjectionStoreV2)({ |
| 887 | getRecoveryThreadIds: () => Effect.succeed([threadId]), |
| 888 | getRuntimeRecoveryProjection: () => Effect.succeed(projection), |
| 889 | }), |
| 890 | Layer.mock(EventSink.EventSinkV2)({ |
| 891 | commitCommand: (input) => { |
| 892 | committedInput = input; |
| 893 | return Effect.succeed({ committed: true, cancelledEffectCount: 2 } as never); |
| 894 | }, |
| 895 | }), |
| 896 | IdAllocator.layer, |
| 897 | Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ |
| 898 | runRecoveryOnce: Effect.succeed(false), |
| 899 | }), |
| 900 | Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 901 | reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), |
| 902 | }), |
| 903 | ), |
| 904 | ), |
| 905 | ); |
| 906 | |
| 907 | return Effect.gen(function* () { |
| 908 | const summary = yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService) |
| 909 | .recover; |
| 910 | assert.equal(summary.terminalizedRuns, 1); |
| 911 | assert.equal(summary.stoppedSessions, 1); |
| 912 | assert.equal(summary.retiredEffects, 2); |
| 913 | assert.deepEqual(committedInput?.cancelUnsettledEffects?.effectTypes, [ |
| 914 | "provider-session.apply-model-selection", |
| 915 | "provider-session.apply-thread-title", |
| 916 | "provider-turn.start", |
| 917 | "provider-turn.interrupt", |
| 918 | "provider-turn.steer", |
| 919 | "provider-turn.restart", |
| 920 | "runtime-request.respond", |
| 921 | ]); |
| 922 | const events = committedInput?.events ?? []; |
| 923 | assert.deepEqual( |
| 924 | events.map((event) => [ |
| 925 | event.type, |
| 926 | "status" in event.payload ? event.payload.status : null, |
| 927 | ]), |
| 928 | [ |
| 929 | ["run.updated", "cancelled"], |
| 930 | ["run-attempt.updated", "cancelled"], |
| 931 | ["node.updated", "cancelled"], |
| 932 | ["provider-turn.updated", "cancelled"], |
| 933 | ["message.updated", null], |
| 934 | ["turn-item.updated", "cancelled"], |
| 935 | ["turn-item.updated", "cancelled"], |
| 936 | ["turn-item.updated", "cancelled"], |
| 937 | ["turn-item.updated", "cancelled"], |
| 938 | ["turn-item.updated", "completed"], |
| 939 | ["provider-thread.updated", "idle"], |
| 940 | ["provider-session.updated", "stopped"], |
| 941 | ], |
| 942 | ); |
| 943 | const messageEvent = events.find((event) => event.type === "message.updated"); |
| 944 | assert.isFalse(messageEvent?.type === "message.updated" && messageEvent.payload.streaming); |
| 945 | const stoppedSession = events.find((event) => event.type === "provider-session.updated"); |
| 946 | assert.deepEqual( |
| 947 | stoppedSession?.type === "provider-session.updated" |
| 948 | ? stoppedSession.payload.extensionStatuses |
| 949 | : undefined, |
| 950 | [], |
| 951 | ); |
| 952 | assert.deepEqual( |
| 953 | events.flatMap((event) => |
| 954 | event.type === "turn-item.updated" && "streaming" in event.payload |
| 955 | ? [[event.payload.type, event.payload.streaming]] |
| 956 | : [], |
| 957 | ), |
| 958 | [ |
| 959 | ["assistant_message", false], |
| 960 | ["reasoning", false], |
| 961 | ["proposed_plan", false], |
| 962 | ], |
| 963 | ); |
| 964 | }).pipe(Effect.provide(layer)); |
| 965 | }, |
| 966 | ); |
| 967 | |
| 968 | it.effect("cancels workflow members and child roots without run ids after restart", () => { |
| 969 | const parentThreadId = ThreadId.make("thread_recovery_workflow_members"); |
| 970 | const childThreadId = ThreadId.make("thread_recovery_workflow_member_child"); |
| 971 | const memberId = NodeId.make("node_recovery_workflow_member"); |
| 972 | const childRootId = NodeId.make("node_recovery_workflow_member_root"); |
| 973 | const providerInstanceId = ProviderInstanceId.make("claudeAgent"); |
| 974 | const driver = ProviderDriverKind.make("claudeAgent"); |
| 975 | const member = { |
| 976 | id: memberId, |
| 977 | threadId: parentThreadId, |
| 978 | runId: null, |
| 979 | origin: "provider_native", |
| 980 | status: "running", |
| 981 | driver, |
| 982 | providerInstanceId, |
| 983 | }; |
| 984 | const parentProjection = { |
| 985 | thread: { id: parentThreadId, providerInstanceId }, |
| 986 | runs: [], |
| 987 | attempts: [], |
| 988 | nodes: [ |
| 989 | { id: memberId, threadId: parentThreadId, runId: null, kind: "subagent", status: "running" }, |
| 990 | ], |
| 991 | subagents: [member], |
| 992 | runtimeRequests: [], |
| 993 | providerSessions: [], |
| 994 | providerThreads: [], |
| 995 | providerTurns: [], |
| 996 | messages: [], |
| 997 | turnItems: [], |
| 998 | } as unknown as OrchestrationV2ThreadProjection; |
| 999 | const childProjection = { |
| 1000 | ...parentProjection, |
| 1001 | thread: { id: childThreadId, providerInstanceId }, |
| 1002 | nodes: [ |
| 1003 | { |
| 1004 | id: childRootId, |
| 1005 | threadId: childThreadId, |
| 1006 | runId: null, |
| 1007 | kind: "root_turn", |
| 1008 | status: "running", |
| 1009 | }, |
| 1010 | ], |
| 1011 | subagents: [], |
| 1012 | } as unknown as OrchestrationV2ThreadProjection; |
| 1013 | const committed: Array<Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0]> = []; |
| 1014 | const layer = ProviderRuntimeRecovery.layer.pipe( |
| 1015 | Layer.provide(ServerSettings.layerTest()), |
| 1016 | Layer.provide( |
| 1017 | Layer.mergeAll( |
| 1018 | Layer.mock(ProjectionStore.ProjectionStoreV2)({ |
| 1019 | getRecoveryThreadIds: () => Effect.succeed([parentThreadId, childThreadId]), |
| 1020 | getRuntimeRecoveryProjection: (threadId) => |
| 1021 | Effect.succeed(threadId === parentThreadId ? parentProjection : childProjection), |
| 1022 | }), |
| 1023 | Layer.mock(EventSink.EventSinkV2)({ |
| 1024 | commitCommand: (input) => { |
| 1025 | committed.push(input); |
| 1026 | return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); |
| 1027 | }, |
| 1028 | }), |
| 1029 | IdAllocator.layer, |
| 1030 | Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ |
| 1031 | runRecoveryOnce: Effect.succeed(false), |
| 1032 | }), |
| 1033 | Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 1034 | reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), |
| 1035 | }), |
| 1036 | ), |
| 1037 | ), |
| 1038 | ); |
| 1039 | |
| 1040 | return Effect.gen(function* () { |
| 1041 | yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); |
| 1042 | assert.deepEqual( |
| 1043 | committed.flatMap((command) => |
| 1044 | command.events.map((event) => [ |
| 1045 | command.threadId, |
| 1046 | event.type, |
| 1047 | "status" in event.payload ? event.payload.status : null, |
| 1048 | ]), |
| 1049 | ), |
| 1050 | [ |
| 1051 | [parentThreadId, "subagent.updated", "cancelled"], |
| 1052 | [parentThreadId, "node.updated", "cancelled"], |
| 1053 | [childThreadId, "node.updated", "cancelled"], |
| 1054 | ], |
| 1055 | ); |
| 1056 | }).pipe(Effect.provide(layer)); |
| 1057 | }); |
| 1058 | |
| 1059 | it.effect( |
| 1060 | "clears persisted pendingBackgroundTasks and terminalizes stale background items on settled runs", |
| 1061 | () => { |
| 1062 | const threadId = ThreadId.make("thread_recovery_background"); |
| 1063 | const settledRunId = RunId.make("run_recovery_background_settled"); |
| 1064 | const activeRunId = RunId.make("run_recovery_background_active"); |
| 1065 | const activeAttemptId = RunAttemptId.make("attempt_recovery_background_active"); |
| 1066 | const activeRootNodeId = NodeId.make("node_recovery_background_active"); |
| 1067 | const idleProviderThreadId = ProviderThreadId.make("provider_thread_recovery_background_idle"); |
| 1068 | const activeProviderThreadId = ProviderThreadId.make( |
| 1069 | "provider_thread_recovery_background_active", |
| 1070 | ); |
| 1071 | const secondaryProviderThreadId = ProviderThreadId.make( |
| 1072 | "provider_thread_recovery_background_secondary", |
| 1073 | ); |
| 1074 | const providerSessionId = ProviderSessionId.make("provider_session_recovery_background"); |
| 1075 | const settledStaleItemId = TurnItemId.make("turn_item_recovery_background_stale"); |
| 1076 | const activeRunItemId = TurnItemId.make("turn_item_recovery_background_active"); |
| 1077 | const nullRunCommandItemId = TurnItemId.make("turn_item_recovery_background_null_run"); |
| 1078 | const nullRunSubagentItemId = TurnItemId.make("turn_item_recovery_background_null_subagent"); |
| 1079 | const claudeInstanceId = ProviderInstanceId.make("claude"); |
| 1080 | const secondaryInstanceId = ProviderInstanceId.make("claude-secondary"); |
| 1081 | const subagentInstanceId = ProviderInstanceId.make("claude-subagent"); |
| 1082 | let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null = |
| 1083 | null; |
| 1084 | const projection = { |
| 1085 | thread: { id: threadId }, |
| 1086 | runtimeRequests: [], |
| 1087 | providerSessions: [ |
| 1088 | { |
| 1089 | id: providerSessionId, |
| 1090 | driver: ProviderDriverKind.make("claude"), |
| 1091 | providerInstanceId: claudeInstanceId, |
| 1092 | status: "ready", |
| 1093 | }, |
| 1094 | ], |
| 1095 | providerThreads: [ |
| 1096 | { |
| 1097 | id: idleProviderThreadId, |
| 1098 | driver: ProviderDriverKind.make("claude"), |
| 1099 | |
| 1100 | providerInstanceId: claudeInstanceId, |
| 1101 | status: "idle", |
| 1102 | pendingBackgroundTasks: [{ taskId: "bg-settled", description: "sleep 30" }], |
| 1103 | }, |
| 1104 | { |
| 1105 | id: activeProviderThreadId, |
| 1106 | driver: ProviderDriverKind.make("claude"), |
| 1107 | providerInstanceId: claudeInstanceId, |
| 1108 | status: "active", |
| 1109 | pendingBackgroundTasks: [{ taskId: "bg-active", description: "npm test" }], |
| 1110 | }, |
| 1111 | { |
| 1112 | id: secondaryProviderThreadId, |
| 1113 | driver: ProviderDriverKind.make("claude"), |
| 1114 | providerInstanceId: secondaryInstanceId, |
| 1115 | status: "idle", |
| 1116 | pendingBackgroundTasks: [], |
| 1117 | }, |
| 1118 | ], |
| 1119 | providerTurns: [{ runAttemptId: activeAttemptId, status: "running" }], |
| 1120 | runs: [ |
| 1121 | { |
| 1122 | id: settledRunId, |
| 1123 | status: "completed", |
| 1124 | providerInstanceId: claudeInstanceId, |
| 1125 | }, |
| 1126 | { |
| 1127 | id: activeRunId, |
| 1128 | status: "running", |
| 1129 | activeAttemptId, |
| 1130 | providerInstanceId: claudeInstanceId, |
| 1131 | }, |
| 1132 | ], |
| 1133 | attempts: [ |
| 1134 | { |
| 1135 | id: activeAttemptId, |
| 1136 | runId: activeRunId, |
| 1137 | rootNodeId: activeRootNodeId, |
| 1138 | status: "running", |
| 1139 | }, |
| 1140 | ], |
| 1141 | nodes: [{ id: activeRootNodeId, runId: activeRunId, status: "running" }], |
| 1142 | subagents: [], |
| 1143 | messages: [], |
| 1144 | turnItems: [ |
| 1145 | { |
| 1146 | id: settledStaleItemId, |
| 1147 | runId: settledRunId, |
| 1148 | nodeId: null, |
| 1149 | providerThreadId: idleProviderThreadId, |
| 1150 | type: "command_execution", |
| 1151 | status: "running", |
| 1152 | }, |
| 1153 | { |
| 1154 | id: activeRunItemId, |
| 1155 | runId: activeRunId, |
| 1156 | nodeId: activeRootNodeId, |
| 1157 | providerThreadId: activeProviderThreadId, |
| 1158 | type: "dynamic_tool", |
| 1159 | status: "running", |
| 1160 | }, |
| 1161 | { |
| 1162 | id: nullRunCommandItemId, |
| 1163 | runId: null, |
| 1164 | nodeId: null, |
| 1165 | providerThreadId: secondaryProviderThreadId, |
| 1166 | type: "command_execution", |
| 1167 | status: "running", |
| 1168 | }, |
| 1169 | { |
| 1170 | id: nullRunSubagentItemId, |
| 1171 | runId: null, |
| 1172 | nodeId: null, |
| 1173 | providerThreadId: secondaryProviderThreadId, |
| 1174 | type: "subagent", |
| 1175 | status: "running", |
| 1176 | providerInstanceId: subagentInstanceId, |
| 1177 | }, |
| 1178 | ], |
| 1179 | } as unknown as OrchestrationV2ThreadProjection; |
| 1180 | const layer = ProviderRuntimeRecovery.layer.pipe( |
| 1181 | Layer.provide(ServerSettings.layerTest()), |
| 1182 | Layer.provide( |
| 1183 | Layer.mergeAll( |
| 1184 | Layer.mock(ProjectionStore.ProjectionStoreV2)({ |
| 1185 | getRecoveryThreadIds: () => Effect.succeed([threadId]), |
| 1186 | getRuntimeRecoveryProjection: () => Effect.succeed(projection), |
| 1187 | }), |
| 1188 | Layer.mock(EventSink.EventSinkV2)({ |
| 1189 | commitCommand: (input) => { |
| 1190 | committedInput = input; |
| 1191 | return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); |
| 1192 | }, |
| 1193 | }), |
| 1194 | IdAllocator.layer, |
| 1195 | Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ |
| 1196 | runRecoveryOnce: Effect.succeed(false), |
| 1197 | }), |
| 1198 | Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 1199 | listByCommandId: () => Effect.succeed([]), |
| 1200 | reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), |
| 1201 | }), |
| 1202 | ), |
| 1203 | ), |
| 1204 | ); |
| 1205 | |
| 1206 | return Effect.gen(function* () { |
| 1207 | const summary = |
| 1208 | yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); |
| 1209 | assert.equal(summary.terminalizedRuns, 1); |
| 1210 | const events = committedInput?.events ?? []; |
| 1211 | |
| 1212 | const turnItemCancels = events.filter( |
| 1213 | (event) => event.type === "turn-item.updated" && event.payload.status === "cancelled", |
| 1214 | ); |
| 1215 | |
| 1216 | assert.equal(turnItemCancels.length, 4); |
| 1217 | assert.deepEqual( |
| 1218 | turnItemCancels |
| 1219 | .map((event) => event.type === "turn-item.updated" && event.payload.id) |
| 1220 | .sort(), |
| 1221 | [activeRunItemId, nullRunCommandItemId, nullRunSubagentItemId, settledStaleItemId].sort(), |
| 1222 | ); |
| 1223 | |
| 1224 | const cancelById = (id: TurnItemId) => |
| 1225 | turnItemCancels.find( |
| 1226 | (event) => event.type === "turn-item.updated" && event.payload.id === id, |
| 1227 | ); |
| 1228 | assert.equal(cancelById(nullRunCommandItemId)?.providerInstanceId, secondaryInstanceId); |
| 1229 | |
| 1230 | assert.notEqual(subagentInstanceId, secondaryInstanceId); |
| 1231 | assert.equal(cancelById(nullRunSubagentItemId)?.providerInstanceId, subagentInstanceId); |
| 1232 | |
| 1233 | assert.equal(cancelById(settledStaleItemId)?.providerInstanceId, claudeInstanceId); |
| 1234 | |
| 1235 | const providerThreadEvents = events.filter( |
| 1236 | (event) => event.type === "provider-thread.updated", |
| 1237 | ); |
| 1238 | |
| 1239 | assert.equal(providerThreadEvents.length, 2); |
| 1240 | for (const event of providerThreadEvents) { |
| 1241 | if (event.type !== "provider-thread.updated") continue; |
| 1242 | assert.deepEqual(event.payload.pendingBackgroundTasks ?? [], []); |
| 1243 | } |
| 1244 | const idleThreadEvent = providerThreadEvents.find( |
| 1245 | (event) => |
| 1246 | event.type === "provider-thread.updated" && event.payload.id === idleProviderThreadId, |
| 1247 | ); |
| 1248 | assert.equal( |
| 1249 | idleThreadEvent?.type === "provider-thread.updated" ? idleThreadEvent.payload.status : null, |
| 1250 | "idle", |
| 1251 | ); |
| 1252 | const activeThreadEvent = providerThreadEvents.find( |
| 1253 | (event) => |
| 1254 | event.type === "provider-thread.updated" && event.payload.id === activeProviderThreadId, |
| 1255 | ); |
| 1256 | assert.equal( |
| 1257 | activeThreadEvent?.type === "provider-thread.updated" |
| 1258 | ? activeThreadEvent.payload.status |
| 1259 | : null, |
| 1260 | "idle", |
| 1261 | ); |
| 1262 | }).pipe(Effect.provide(layer)); |
| 1263 | }, |
| 1264 | ); |
| 1265 | |
| 1266 | it.effect( |
| 1267 | "terminalizes a leftover nonpersistent dynamic_tool on a settled run after process loss", |
| 1268 | () => { |
| 1269 | const threadId = ThreadId.make("thread_recovery_orphan_wait"); |
| 1270 | const settledRunId = RunId.make("run_recovery_orphan_wait_settled"); |
| 1271 | const providerThreadId = ProviderThreadId.make("provider_thread_recovery_orphan_wait"); |
| 1272 | const orphanWaitItemId = TurnItemId.make( |
| 1273 | "turn-item:provider:codex:native-item:exec-4669f3bb-78c9-4af1-b44e-daa340d2c538", |
| 1274 | ); |
| 1275 | const persistentMonitorItemId = TurnItemId.make( |
| 1276 | "turn-item:provider:codex:native-item:exec-persistent-monitor", |
| 1277 | ); |
| 1278 | const orphanWaitNodeId = NodeId.make("node_recovery_orphan_wait"); |
| 1279 | const persistentMonitorNodeId = NodeId.make("node_recovery_persistent_monitor"); |
| 1280 | const codexInstanceId = ProviderInstanceId.make("codex"); |
| 1281 | let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null = |
| 1282 | null; |
| 1283 | const projection = { |
| 1284 | thread: { id: threadId, providerInstanceId: codexInstanceId }, |
| 1285 | runtimeRequests: [], |
| 1286 | providerSessions: [], |
| 1287 | providerThreads: [ |
| 1288 | { |
| 1289 | id: providerThreadId, |
| 1290 | driver: ProviderDriverKind.make("codex"), |
| 1291 | providerInstanceId: codexInstanceId, |
| 1292 | status: "idle", |
| 1293 | pendingBackgroundTasks: [], |
| 1294 | }, |
| 1295 | ], |
| 1296 | providerTurns: [], |
| 1297 | runs: [{ id: settledRunId, status: "completed", providerInstanceId: codexInstanceId }], |
| 1298 | attempts: [], |
| 1299 | nodes: [ |
| 1300 | { id: orphanWaitNodeId, runId: settledRunId, status: "running", kind: "tool_call" }, |
| 1301 | { |
| 1302 | id: persistentMonitorNodeId, |
| 1303 | runId: settledRunId, |
| 1304 | status: "running", |
| 1305 | kind: "tool_call", |
| 1306 | }, |
| 1307 | ], |
| 1308 | subagents: [], |
| 1309 | messages: [], |
| 1310 | turnItems: [ |
| 1311 | { |
| 1312 | id: orphanWaitItemId, |
| 1313 | runId: settledRunId, |
| 1314 | nodeId: orphanWaitNodeId, |
| 1315 | providerThreadId, |
| 1316 | type: "dynamic_tool", |
| 1317 | status: "running", |
| 1318 | toolName: "t3-code.t3_thread_wait", |
| 1319 | input: { |
| 1320 | threadId: |
| 1321 | "thread:delegated-task:command%3Amcp%3Aaafffab1-e811-458a-ae83-558e542c61ff%3Adelegate-task%3Areview-mobile-reconnect-opus-20260815", |
| 1322 | timeoutMs: 30000, |
| 1323 | }, |
| 1324 | }, |
| 1325 | { |
| 1326 | id: persistentMonitorItemId, |
| 1327 | runId: settledRunId, |
| 1328 | nodeId: persistentMonitorNodeId, |
| 1329 | providerThreadId, |
| 1330 | type: "dynamic_tool", |
| 1331 | status: "running", |
| 1332 | toolName: "grok.monitor", |
| 1333 | input: { persistent: true, command: "tail -f" }, |
| 1334 | }, |
| 1335 | ], |
| 1336 | } as unknown as OrchestrationV2ThreadProjection; |
| 1337 | const layer = ProviderRuntimeRecovery.layer.pipe( |
| 1338 | Layer.provide(ServerSettings.layerTest()), |
| 1339 | Layer.provide( |
| 1340 | Layer.mergeAll( |
| 1341 | Layer.mock(ProjectionStore.ProjectionStoreV2)({ |
| 1342 | getRecoveryThreadIds: () => Effect.succeed([threadId]), |
| 1343 | getRuntimeRecoveryProjection: () => Effect.succeed(projection), |
| 1344 | }), |
| 1345 | Layer.mock(EventSink.EventSinkV2)({ |
| 1346 | commitCommand: (input) => { |
| 1347 | committedInput = input; |
| 1348 | return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); |
| 1349 | }, |
| 1350 | }), |
| 1351 | IdAllocator.layer, |
| 1352 | Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ |
| 1353 | runRecoveryOnce: Effect.succeed(false), |
| 1354 | }), |
| 1355 | Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 1356 | listByCommandId: () => Effect.succeed([]), |
| 1357 | reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), |
| 1358 | }), |
| 1359 | ), |
| 1360 | ), |
| 1361 | ); |
| 1362 | |
| 1363 | return Effect.gen(function* () { |
| 1364 | const summary = |
| 1365 | yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); |
| 1366 | assert.equal(summary.terminalizedRuns, 0); |
| 1367 | const turnItemCancels = (committedInput?.events ?? []).filter( |
| 1368 | (event) => event.type === "turn-item.updated" && event.payload.status === "cancelled", |
| 1369 | ); |
| 1370 | |
| 1371 | assert.equal(turnItemCancels.length, 2); |
| 1372 | assert.deepEqual( |
| 1373 | turnItemCancels |
| 1374 | .map((event) => event.type === "turn-item.updated" && event.payload.id) |
| 1375 | .sort(), |
| 1376 | [orphanWaitItemId, persistentMonitorItemId].sort(), |
| 1377 | ); |
| 1378 | const nodeCancels = (committedInput?.events ?? []).filter( |
| 1379 | (event) => event.type === "node.updated" && event.payload.status === "cancelled", |
| 1380 | ); |
| 1381 | assert.equal(nodeCancels.length, 2); |
| 1382 | assert.deepEqual( |
| 1383 | nodeCancels.map((event) => event.type === "node.updated" && event.payload.id).sort(), |
| 1384 | [orphanWaitNodeId, persistentMonitorNodeId].sort(), |
| 1385 | ); |
| 1386 | }).pipe(Effect.provide(layer)); |
| 1387 | }, |
| 1388 | ); |
| 1389 | |
| 1390 | it.effect( |
| 1391 | "terminalizes the linked subagent and node for a stale subagent item on a settled run", |
| 1392 | () => { |
| 1393 | const threadId = ThreadId.make("thread_recovery_subagent"); |
| 1394 | const settledRunId = RunId.make("run_recovery_subagent_settled"); |
| 1395 | const providerThreadId = ProviderThreadId.make("provider_thread_recovery_subagent"); |
| 1396 | const staleSubagentNodeId = NodeId.make("node_recovery_subagent_stale"); |
| 1397 | const doneSubagentNodeId = NodeId.make("node_recovery_subagent_done"); |
| 1398 | const staleItemId = TurnItemId.make("turn_item_recovery_subagent_stale"); |
| 1399 | const doneItemId = TurnItemId.make("turn_item_recovery_subagent_done"); |
| 1400 | const claudeInstanceId = ProviderInstanceId.make("claude"); |
| 1401 | let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null = |
| 1402 | null; |
| 1403 | const projection = { |
| 1404 | thread: { id: threadId }, |
| 1405 | runtimeRequests: [], |
| 1406 | providerSessions: [], |
| 1407 | providerThreads: [ |
| 1408 | { |
| 1409 | id: providerThreadId, |
| 1410 | driver: ProviderDriverKind.make("claude"), |
| 1411 | providerInstanceId: claudeInstanceId, |
| 1412 | status: "idle", |
| 1413 | pendingBackgroundTasks: [], |
| 1414 | }, |
| 1415 | ], |
| 1416 | providerTurns: [], |
| 1417 | |
| 1418 | runs: [{ id: settledRunId, status: "completed", providerInstanceId: claudeInstanceId }], |
| 1419 | attempts: [], |
| 1420 | nodes: [ |
| 1421 | { id: staleSubagentNodeId, runId: settledRunId, status: "running" }, |
| 1422 | { id: doneSubagentNodeId, runId: settledRunId, status: "completed" }, |
| 1423 | ], |
| 1424 | subagents: [ |
| 1425 | { |
| 1426 | id: staleSubagentNodeId, |
| 1427 | runId: settledRunId, |
| 1428 | origin: "provider_native", |
| 1429 | driver: ProviderDriverKind.make("claude"), |
| 1430 | providerInstanceId: claudeInstanceId, |
| 1431 | status: "running", |
| 1432 | }, |
| 1433 | { |
| 1434 | id: doneSubagentNodeId, |
| 1435 | runId: settledRunId, |
| 1436 | origin: "provider_native", |
| 1437 | driver: ProviderDriverKind.make("claude"), |
| 1438 | providerInstanceId: claudeInstanceId, |
| 1439 | status: "completed", |
| 1440 | result: "done", |
| 1441 | }, |
| 1442 | ], |
| 1443 | messages: [], |
| 1444 | turnItems: [ |
| 1445 | { |
| 1446 | id: staleItemId, |
| 1447 | runId: settledRunId, |
| 1448 | nodeId: staleSubagentNodeId, |
| 1449 | providerThreadId, |
| 1450 | type: "subagent", |
| 1451 | status: "running", |
| 1452 | subagentId: staleSubagentNodeId, |
| 1453 | providerInstanceId: claudeInstanceId, |
| 1454 | }, |
| 1455 | { |
| 1456 | id: doneItemId, |
| 1457 | runId: settledRunId, |
| 1458 | nodeId: doneSubagentNodeId, |
| 1459 | providerThreadId, |
| 1460 | type: "subagent", |
| 1461 | status: "completed", |
| 1462 | subagentId: doneSubagentNodeId, |
| 1463 | providerInstanceId: claudeInstanceId, |
| 1464 | }, |
| 1465 | ], |
| 1466 | } as unknown as OrchestrationV2ThreadProjection; |
| 1467 | const layer = ProviderRuntimeRecovery.layer.pipe( |
| 1468 | Layer.provide(ServerSettings.layerTest()), |
| 1469 | Layer.provide( |
| 1470 | Layer.mergeAll( |
| 1471 | Layer.mock(ProjectionStore.ProjectionStoreV2)({ |
| 1472 | getRecoveryThreadIds: () => Effect.succeed([threadId]), |
| 1473 | getRuntimeRecoveryProjection: () => Effect.succeed(projection), |
| 1474 | }), |
| 1475 | Layer.mock(EventSink.EventSinkV2)({ |
| 1476 | commitCommand: (input) => { |
| 1477 | committedInput = input; |
| 1478 | return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); |
| 1479 | }, |
| 1480 | }), |
| 1481 | IdAllocator.layer, |
| 1482 | Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ |
| 1483 | runRecoveryOnce: Effect.succeed(false), |
| 1484 | }), |
| 1485 | Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 1486 | listByCommandId: () => Effect.succeed([]), |
| 1487 | reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), |
| 1488 | }), |
| 1489 | ), |
| 1490 | ), |
| 1491 | ); |
| 1492 | |
| 1493 | return Effect.gen(function* () { |
| 1494 | yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); |
| 1495 | const events = committedInput?.events ?? []; |
| 1496 | |
| 1497 | const turnItemCancels = events.filter( |
| 1498 | (event) => event.type === "turn-item.updated" && event.payload.status === "cancelled", |
| 1499 | ); |
| 1500 | assert.equal(turnItemCancels.length, 1); |
| 1501 | |
| 1502 | const subagentCancels = events.filter((event) => event.type === "subagent.updated"); |
| 1503 | assert.equal(subagentCancels.length, 1); |
| 1504 | const subagentCancel = subagentCancels[0]; |
| 1505 | assert.equal( |
| 1506 | subagentCancel?.type === "subagent.updated" ? subagentCancel.payload.id : null, |
| 1507 | staleSubagentNodeId, |
| 1508 | ); |
| 1509 | assert.equal( |
| 1510 | subagentCancel?.type === "subagent.updated" ? subagentCancel.payload.status : null, |
| 1511 | "cancelled", |
| 1512 | ); |
| 1513 | |
| 1514 | const nodeCancels = events.filter((event) => event.type === "node.updated"); |
| 1515 | assert.equal(nodeCancels.length, 1); |
| 1516 | const nodeCancel = nodeCancels[0]; |
| 1517 | assert.equal( |
| 1518 | nodeCancel?.type === "node.updated" ? nodeCancel.payload.id : null, |
| 1519 | staleSubagentNodeId, |
| 1520 | ); |
| 1521 | |
| 1522 | assert.isFalse( |
| 1523 | events.some( |
| 1524 | (event) => event.type === "subagent.updated" && event.payload.id === doneSubagentNodeId, |
| 1525 | ), |
| 1526 | ); |
| 1527 | assert.isFalse( |
| 1528 | events.some( |
| 1529 | (event) => event.type === "node.updated" && event.payload.id === doneSubagentNodeId, |
| 1530 | ), |
| 1531 | ); |
| 1532 | }).pipe(Effect.provide(layer)); |
| 1533 | }, |
| 1534 | ); |
| 1535 | |
| 1536 | it.effect("leaves delegated tasks to their own child threads after process loss", () => { |
| 1537 | const threadId = ThreadId.make("thread_recovery_delegation"); |
| 1538 | const settledRunId = RunId.make("run_recovery_delegation_settled"); |
| 1539 | const runningRunId = RunId.make("run_recovery_delegation_running"); |
| 1540 | const providerThreadId = ProviderThreadId.make("provider_thread_recovery_delegation"); |
| 1541 | const claudeInstanceId = ProviderInstanceId.make("claude"); |
| 1542 | const driver = ProviderDriverKind.make("claude"); |
| 1543 | const settledTaskId = NodeId.make("node_recovery_delegation_settled"); |
| 1544 | const runningTaskId = NodeId.make("node_recovery_delegation_running"); |
| 1545 | const nativeSubagentId = NodeId.make("node_recovery_native_subagent"); |
| 1546 | const delegatedTask = (id: NodeId, runId: RunId) => ({ |
| 1547 | id, |
| 1548 | runId, |
| 1549 | origin: "app_owned", |
| 1550 | childThreadId: ThreadId.make(`thread:delegated-task:${id}`), |
| 1551 | driver, |
| 1552 | providerInstanceId: claudeInstanceId, |
| 1553 | status: "running", |
| 1554 | }); |
| 1555 | const subagentItem = (subagentId: NodeId, runId: RunId, origin: string) => ({ |
| 1556 | id: TurnItemId.make(`turn_item:${subagentId}`), |
| 1557 | runId, |
| 1558 | nodeId: subagentId, |
| 1559 | providerThreadId, |
| 1560 | type: "subagent", |
| 1561 | status: "running", |
| 1562 | subagentId, |
| 1563 | origin, |
| 1564 | childThreadId: |
| 1565 | origin === "app_owned" ? ThreadId.make(`thread:delegated-task:${subagentId}`) : null, |
| 1566 | providerInstanceId: claudeInstanceId, |
| 1567 | title: `subagent ${subagentId}`, |
| 1568 | }); |
| 1569 | let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null = |
| 1570 | null; |
| 1571 | const projection = { |
| 1572 | thread: { id: threadId, providerInstanceId: claudeInstanceId }, |
| 1573 | runtimeRequests: [], |
| 1574 | providerSessions: [], |
| 1575 | providerThreads: [ |
| 1576 | { |
| 1577 | id: providerThreadId, |
| 1578 | driver, |
| 1579 | providerInstanceId: claudeInstanceId, |
| 1580 | ownerNodeId: null, |
| 1581 | status: "idle", |
| 1582 | pendingBackgroundTasks: [], |
| 1583 | }, |
| 1584 | ], |
| 1585 | providerTurns: [ |
| 1586 | { runAttemptId: RunAttemptId.make("attempt_recovery_delegation"), status: "running" }, |
| 1587 | ], |
| 1588 | runs: [ |
| 1589 | { |
| 1590 | id: settledRunId, |
| 1591 | ordinal: 1, |
| 1592 | status: "completed", |
| 1593 | providerThreadId, |
| 1594 | providerInstanceId: claudeInstanceId, |
| 1595 | }, |
| 1596 | { |
| 1597 | id: runningRunId, |
| 1598 | ordinal: 2, |
| 1599 | status: "running", |
| 1600 | activeAttemptId: RunAttemptId.make("attempt_recovery_delegation"), |
| 1601 | providerThreadId, |
| 1602 | providerInstanceId: claudeInstanceId, |
| 1603 | }, |
| 1604 | ], |
| 1605 | attempts: [], |
| 1606 | nodes: [ |
| 1607 | { id: settledTaskId, runId: settledRunId, status: "running", kind: "subagent" }, |
| 1608 | { id: runningTaskId, runId: runningRunId, status: "running", kind: "subagent" }, |
| 1609 | { id: nativeSubagentId, runId: settledRunId, status: "running", kind: "subagent" }, |
| 1610 | ], |
| 1611 | subagents: [ |
| 1612 | delegatedTask(settledTaskId, settledRunId), |
| 1613 | delegatedTask(runningTaskId, runningRunId), |
| 1614 | { |
| 1615 | ...delegatedTask(nativeSubagentId, settledRunId), |
| 1616 | origin: "provider_native", |
| 1617 | childThreadId: null, |
| 1618 | }, |
| 1619 | ], |
| 1620 | messages: [], |
| 1621 | turnItems: [ |
| 1622 | subagentItem(settledTaskId, settledRunId, "app_owned"), |
| 1623 | subagentItem(runningTaskId, runningRunId, "app_owned"), |
| 1624 | subagentItem(nativeSubagentId, settledRunId, "provider_native"), |
| 1625 | ], |
| 1626 | } as unknown as OrchestrationV2ThreadProjection; |
| 1627 | const layer = ProviderRuntimeRecovery.layer.pipe( |
| 1628 | Layer.provide(ServerSettings.layerTest()), |
| 1629 | Layer.provide( |
| 1630 | Layer.mergeAll( |
| 1631 | Layer.mock(ProjectionStore.ProjectionStoreV2)({ |
| 1632 | getRecoveryThreadIds: () => Effect.succeed([threadId]), |
| 1633 | getRuntimeRecoveryProjection: () => Effect.succeed(projection), |
| 1634 | }), |
| 1635 | Layer.mock(EventSink.EventSinkV2)({ |
| 1636 | commitCommand: (input) => { |
| 1637 | committedInput = input; |
| 1638 | return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); |
| 1639 | }, |
| 1640 | }), |
| 1641 | IdAllocator.layer, |
| 1642 | Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ |
| 1643 | runRecoveryOnce: Effect.succeed(false), |
| 1644 | }), |
| 1645 | Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 1646 | listByCommandId: () => Effect.succeed([]), |
| 1647 | reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), |
| 1648 | }), |
| 1649 | ), |
| 1650 | ), |
| 1651 | ); |
| 1652 | |
| 1653 | return Effect.gen(function* () { |
| 1654 | yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); |
| 1655 | const events = committedInput?.events ?? []; |
| 1656 | const touched = events.flatMap((event) => |
| 1657 | event.type === "turn-item.updated" |
| 1658 | ? event.payload.type === "subagent" |
| 1659 | ? [event.payload.subagentId] |
| 1660 | : [] |
| 1661 | : event.type === "subagent.updated" || event.type === "node.updated" |
| 1662 | ? [event.payload.id] |
| 1663 | : [], |
| 1664 | ); |
| 1665 | |
| 1666 | assert.isTrue( |
| 1667 | events.some( |
| 1668 | (event) => |
| 1669 | event.type === "run.updated" && |
| 1670 | event.payload.id === runningRunId && |
| 1671 | event.payload.status === "cancelled", |
| 1672 | ), |
| 1673 | ); |
| 1674 | assert.notInclude(touched, settledTaskId); |
| 1675 | assert.notInclude(touched, runningTaskId); |
| 1676 | assert.include(touched, nativeSubagentId); |
| 1677 | |
| 1678 | const noted = events.flatMap((event) => |
| 1679 | event.type === "run.background-work-cancelled" |
| 1680 | ? event.payload.restartCancelledBackgroundWork.map((work) => work.label) |
| 1681 | : [], |
| 1682 | ); |
| 1683 | assert.deepEqual(noted, [`subagent ${nativeSubagentId}`]); |
| 1684 | }).pipe(Effect.provide(layer)); |
| 1685 | }); |