The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | import { assert, it } from "@effect/vitest"; |
| 2 | import { |
| 3 | CheckpointId, |
| 4 | CheckpointScopeId, |
| 5 | CommandId, |
| 6 | NodeId, |
| 7 | ProjectId, |
| 8 | ProviderInstanceId, |
| 9 | ProviderSessionId, |
| 10 | ProviderThreadId, |
| 11 | ProviderTurnId, |
| 12 | RunAttemptId, |
| 13 | RunId, |
| 14 | RuntimeRequestId, |
| 15 | ThreadId, |
| 16 | VcsProcessExitError, |
| 17 | type OrchestrationV2ThreadProjection, |
| 18 | } from "@t3tools/contracts"; |
| 19 | import * as Cause from "effect/Cause"; |
| 20 | import * as DateTime from "effect/DateTime"; |
| 21 | import * as Deferred from "effect/Deferred"; |
| 22 | import * as Duration from "effect/Duration"; |
| 23 | import * as Effect from "effect/Effect"; |
| 24 | import * as Exit from "effect/Exit"; |
| 25 | import * as Fiber from "effect/Fiber"; |
| 26 | import * as Layer from "effect/Layer"; |
| 27 | import * as Option from "effect/Option"; |
| 28 | import * as Ref from "effect/Ref"; |
| 29 | import * as SqlError from "effect/sql/SqlError"; |
| 30 | import * as Stream from "effect/Stream"; |
| 31 | import * as TestClock from "effect/testing/TestClock"; |
| 32 | |
| 33 | import * as CheckpointRollbackService from "./CheckpointRollbackService.ts"; |
| 34 | import * as CheckpointService from "./CheckpointService.ts"; |
| 35 | import * as EffectOutbox from "./EffectOutbox.ts"; |
| 36 | import * as EffectWorker from "./EffectWorker.ts"; |
| 37 | import * as ProjectionStore from "./ProjectionStore.ts"; |
| 38 | import * as RunFinalizationService from "./RunFinalizationService.ts"; |
| 39 | import * as ProviderSessionManager from "./ProviderSessionManager.ts"; |
| 40 | import * as ProviderTurnControlService from "./ProviderTurnControlService.ts"; |
| 41 | import * as ProviderTurnStartService from "./ProviderTurnStartService.ts"; |
| 42 | import * as RuntimeRequestService from "./RuntimeRequestService.ts"; |
| 43 | import * as ThreadTitleRegenerationService from "./ThreadTitleRegenerationService.ts"; |
| 44 | import * as ThreadManagementService from "./ThreadManagementService.ts"; |
| 45 | import * as ServerSettings from "../serverSettings.ts"; |
| 46 | |
| 47 | const threadId = ThreadId.make("thread:effect-worker-restart"); |
| 48 | const oldSessionId = ProviderSessionId.make("provider-session:effect-worker-restart:old"); |
| 49 | const providerThreadId = ProviderThreadId.make("provider-thread:effect-worker-restart"); |
| 50 | const providerTurnId = ProviderTurnId.make("provider-turn:effect-worker-restart"); |
| 51 | const attemptId = RunAttemptId.make("run-attempt:effect-worker-restart"); |
| 52 | const runId = RunId.make("run:effect-worker-restart"); |
| 53 | |
| 54 | function restartEffect( |
| 55 | now: DateTime.Utc, |
| 56 | sessionTransition: NonNullable< |
| 57 | Extract< |
| 58 | EffectOutbox.OrchestrationEffectV2["request"], |
| 59 | { readonly type: "provider-turn.restart" } |
| 60 | >["sessionTransition"] |
| 61 | >, |
| 62 | ): EffectOutbox.OrchestrationEffectV2 { |
| 63 | const timestamp = DateTime.formatIso(now); |
| 64 | return { |
| 65 | id: `effect:restart:${sessionTransition.type}`, |
| 66 | commandId: CommandId.make(`command:restart:${sessionTransition.type}`), |
| 67 | threadId, |
| 68 | request: { |
| 69 | type: "provider-turn.restart", |
| 70 | providerSessionId: oldSessionId, |
| 71 | providerThreadId, |
| 72 | providerTurnId, |
| 73 | interruptedAttemptId: attemptId, |
| 74 | runId, |
| 75 | sessionTransition, |
| 76 | }, |
| 77 | status: "running", |
| 78 | attemptCount: 1, |
| 79 | availableAt: timestamp, |
| 80 | leaseOwner: "test-worker", |
| 81 | createdAt: timestamp, |
| 82 | updatedAt: timestamp, |
| 83 | completedAt: null, |
| 84 | lastError: null, |
| 85 | }; |
| 86 | } |
| 87 | |
| 88 | function layerExecutorFor(input: { |
| 89 | readonly events: Ref.Ref<ReadonlyArray<string>>; |
| 90 | readonly failFirstStart?: Ref.Ref<boolean>; |
| 91 | readonly threads?: Partial<ThreadManagementService.ThreadManagementService["Service"]>; |
| 92 | readonly continueAfterRestart?: boolean; |
| 93 | readonly interrupt?: ProviderTurnControlService.ProviderTurnControlServiceV2Shape["interrupt"]; |
| 94 | readonly rollback?: CheckpointRollbackService.CheckpointRollbackServiceV2Shape["execute"]; |
| 95 | readonly respond?: RuntimeRequestService.RuntimeRequestServiceV2["Service"]["respond"]; |
| 96 | }) { |
| 97 | const record = (event: string) => Ref.update(input.events, (events) => [...events, event]); |
| 98 | const layerDependencies = Layer.mergeAll( |
| 99 | Layer.succeed( |
| 100 | ProviderTurnControlService.ProviderTurnControlServiceV2, |
| 101 | ProviderTurnControlService.ProviderTurnControlServiceV2.of({ |
| 102 | interrupt: input.interrupt ?? (() => Effect.void), |
| 103 | steer: () => Effect.void, |
| 104 | applyModelSelection: () => Effect.void, |
| 105 | applyThreadTitle: () => Effect.void, |
| 106 | interruptAndAwaitTerminal: (request) => |
| 107 | record( |
| 108 | request.replacementProviderSessionId === undefined |
| 109 | ? "interrupt" |
| 110 | : `interrupt:${request.replacementProviderSessionId}`, |
| 111 | ), |
| 112 | }), |
| 113 | ), |
| 114 | Layer.succeed( |
| 115 | ProviderSessionManager.ProviderSessionManagerV2, |
| 116 | ProviderSessionManager.ProviderSessionManagerV2.of({ |
| 117 | shutdown: Effect.void, |
| 118 | open: () => Effect.die("unused open"), |
| 119 | get: () => Effect.succeedNone, |
| 120 | isLive: () => Effect.succeed(false), |
| 121 | close: () => Effect.void, |
| 122 | closeInstance: () => Effect.void, |
| 123 | release: () => record("release"), |
| 124 | detach: () => record("detach"), |
| 125 | otherWorkingThreads: () => Effect.succeed([]), |
| 126 | releaseUnlessBusy: () => Effect.succeed(true), |
| 127 | forgetLoadedProviderThread: () => Effect.void, |
| 128 | subscribePromptSuggestion: () => Stream.empty, |
| 129 | }), |
| 130 | ), |
| 131 | Layer.succeed( |
| 132 | ProviderTurnStartService.ProviderTurnStartServiceV2, |
| 133 | ProviderTurnStartService.ProviderTurnStartServiceV2.of({ |
| 134 | failStartingRun: () => Effect.void, |
| 135 | start: () => |
| 136 | Effect.gen(function* () { |
| 137 | yield* record("start"); |
| 138 | if ( |
| 139 | input.failFirstStart !== undefined && |
| 140 | (yield* Ref.getAndSet(input.failFirstStart, false)) |
| 141 | ) { |
| 142 | return yield* new ProviderTurnStartService.ProviderTurnStartError({ |
| 143 | runId, |
| 144 | cause: "simulated first start failure", |
| 145 | }); |
| 146 | } |
| 147 | }), |
| 148 | }), |
| 149 | ), |
| 150 | Layer.succeed( |
| 151 | RunFinalizationService.RunFinalizationService, |
| 152 | RunFinalizationService.RunFinalizationService.of({ finalize: () => Effect.void }), |
| 153 | ), |
| 154 | Layer.succeed( |
| 155 | CheckpointRollbackService.CheckpointRollbackServiceV2, |
| 156 | CheckpointRollbackService.CheckpointRollbackServiceV2.of({ |
| 157 | execute: input.rollback ?? (() => Effect.void), |
| 158 | }), |
| 159 | ), |
| 160 | Layer.succeed( |
| 161 | RuntimeRequestService.RuntimeRequestServiceV2, |
| 162 | RuntimeRequestService.RuntimeRequestServiceV2.of({ |
| 163 | respond: input.respond ?? (() => Effect.void), |
| 164 | }), |
| 165 | ), |
| 166 | Layer.succeed( |
| 167 | ThreadTitleRegenerationService.ThreadTitleRegenerationService, |
| 168 | ThreadTitleRegenerationService.ThreadTitleRegenerationService.of({ |
| 169 | execute: () => Effect.void, |
| 170 | }), |
| 171 | ), |
| 172 | ); |
| 173 | return EffectWorker.layerExecutor.pipe( |
| 174 | Layer.provide( |
| 175 | Layer.mergeAll( |
| 176 | layerDependencies, |
| 177 | Layer.mock(ThreadManagementService.ThreadManagementService)(input.threads ?? {}), |
| 178 | Layer.mock(ProjectionStore.ProjectionStoreV2)({}), |
| 179 | ServerSettings.layerTest( |
| 180 | input.continueAfterRestart === true ? { continueThreadsAfterServerUpdate: true } : {}, |
| 181 | ), |
| 182 | ), |
| 183 | ), |
| 184 | ); |
| 185 | } |
| 186 | |
| 187 | it.effect.each([ |
| 188 | { |
| 189 | name: "settles a stopped run when its adapter has already lost the native turn", |
| 190 | unresponsive: false, |
| 191 | dispatched: ["thread.background-work.settle"], |
| 192 | }, |
| 193 | { |
| 194 | name: "leaves the run running and flags the Stop when the provider never confirms it", |
| 195 | unresponsive: true, |
| 196 | dispatched: ["thread.interrupt.unresponsive"], |
| 197 | }, |
| 198 | ])("$name", ({ unresponsive, dispatched }) => |
| 199 | Effect.gen(function* () { |
| 200 | const now = yield* DateTime.now; |
| 201 | const events = yield* Ref.make<ReadonlyArray<string>>([]); |
| 202 | const layer = layerExecutorFor({ |
| 203 | events, |
| 204 | interrupt: () => |
| 205 | new ProviderTurnControlService.ProviderTurnControlError({ |
| 206 | threadId, |
| 207 | operation: "interrupt", |
| 208 | providerTurnId, |
| 209 | ...(unresponsive ? { unresponsive } : {}), |
| 210 | cause: unresponsive ? "Timed out." : "Provider turn is not active.", |
| 211 | }), |
| 212 | threads: { |
| 213 | dispatch: (command) => |
| 214 | Ref.update(events, (current) => [...current, command.type]).pipe( |
| 215 | Effect.as({ sequence: 1, storedEvents: [] }), |
| 216 | ), |
| 217 | }, |
| 218 | }); |
| 219 | yield* Effect.gen(function* () { |
| 220 | const executor = yield* EffectWorker.OrchestrationEffectExecutorV2; |
| 221 | yield* executor.execute({ |
| 222 | ...restartEffect(now, { type: "detach" }), |
| 223 | request: { |
| 224 | type: "provider-turn.interrupt", |
| 225 | providerSessionId: oldSessionId, |
| 226 | providerThreadId, |
| 227 | providerTurnId, |
| 228 | }, |
| 229 | }); |
| 230 | }).pipe(Effect.provide(layer)); |
| 231 | assert.deepEqual(yield* Ref.get(events), dispatched); |
| 232 | }), |
| 233 | ); |
| 234 | |
| 235 | const lockedIndexMessage = |
| 236 | "Git's index is locked. Wait for other Git operations to finish, then try again."; |
| 237 | const rollbackCheckpointId = CheckpointId.make("checkpoint:effect-worker-rollback"); |
| 238 | const rollbackScopeId = CheckpointScopeId.make("checkpoint-scope:effect-worker-rollback"); |
| 239 | const restoreFailure = (detail: string) => |
| 240 | new CheckpointRollbackService.CheckpointRollbackExecutionError({ |
| 241 | reason: "unexpected-failure", |
| 242 | threadId, |
| 243 | providerThreadId, |
| 244 | checkpointId: rollbackCheckpointId, |
| 245 | cause: new CheckpointService.CheckpointRestoreError({ |
| 246 | scopeId: rollbackScopeId, |
| 247 | checkpointId: rollbackCheckpointId, |
| 248 | cause: new VcsProcessExitError({ |
| 249 | operation: "GitVcsDriverCore.restoreCheckpoint", |
| 250 | command: "git", |
| 251 | cwd: "/private/repo", |
| 252 | exitCode: 128, |
| 253 | detail, |
| 254 | failureKind: "command-failed", |
| 255 | }), |
| 256 | }), |
| 257 | }); |
| 258 | |
| 259 | it.effect.each([ |
| 260 | { |
| 261 | name: "a held Git index lock", |
| 262 | failure: restoreFailure(lockedIndexMessage), |
| 263 | message: lockedIndexMessage, |
| 264 | }, |
| 265 | { |
| 266 | name: "raw Git output", |
| 267 | failure: restoreFailure("error: could not clean /private/repo/secret.txt"), |
| 268 | message: CheckpointRollbackService.ROLLBACK_FAILED_MESSAGE, |
| 269 | }, |
| 270 | ])("reports a rollback blocked by $name only after the last attempt", ({ failure, message }) => |
| 271 | Effect.gen(function* () { |
| 272 | const now = yield* DateTime.now; |
| 273 | const failures = yield* Ref.make<ReadonlyArray<string>>([]); |
| 274 | const layer = layerExecutorFor({ |
| 275 | events: yield* Ref.make<ReadonlyArray<string>>([]), |
| 276 | rollback: () => Effect.fail(failure), |
| 277 | threads: { |
| 278 | dispatch: (command) => |
| 279 | Ref.update(failures, (current) => |
| 280 | command.type === "checkpoint.rollback.fail" ? [...current, command.message] : current, |
| 281 | ).pipe(Effect.as({ sequence: 1, storedEvents: [] })), |
| 282 | }, |
| 283 | }); |
| 284 | yield* Effect.gen(function* () { |
| 285 | const executor = yield* EffectWorker.OrchestrationEffectExecutorV2; |
| 286 | const rollback: EffectOutbox.OrchestrationEffectV2 = { |
| 287 | ...restartEffect(now, { type: "detach" }), |
| 288 | request: { |
| 289 | type: "provider-thread.rollback", |
| 290 | providerThreadId, |
| 291 | checkpointId: rollbackCheckpointId, |
| 292 | scopeId: rollbackScopeId, |
| 293 | restoreFiles: true, |
| 294 | }, |
| 295 | }; |
| 296 | yield* Effect.flip(executor.execute(rollback, { willRetry: true })); |
| 297 | assert.deepEqual(yield* Ref.get(failures), []); |
| 298 | yield* Effect.flip(executor.execute(rollback, { willRetry: false })); |
| 299 | }).pipe(Effect.provide(layer)); |
| 300 | assert.deepEqual(yield* Ref.get(failures), [message]); |
| 301 | }), |
| 302 | ); |
| 303 | |
| 304 | it.effect("does not report an answer whose last delivery attempt is interrupted", () => |
| 305 | Effect.gen(function* () { |
| 306 | const now = yield* DateTime.now; |
| 307 | const reported = yield* Ref.make(0); |
| 308 | const layer = layerExecutorFor({ |
| 309 | events: yield* Ref.make<ReadonlyArray<string>>([]), |
| 310 | respond: () => Effect.interrupt, |
| 311 | threads: { |
| 312 | dispatch: (command) => |
| 313 | Ref.update(reported, (count) => |
| 314 | command.type === "runtime-request.delivery.fail" ? count + 1 : count, |
| 315 | ).pipe(Effect.as({ sequence: 1, storedEvents: [] })), |
| 316 | }, |
| 317 | }); |
| 318 | const exit = yield* Effect.gen(function* () { |
| 319 | const executor = yield* EffectWorker.OrchestrationEffectExecutorV2; |
| 320 | return yield* Effect.exit( |
| 321 | executor.execute( |
| 322 | { |
| 323 | ...restartEffect(now, { type: "detach" }), |
| 324 | request: { |
| 325 | type: "runtime-request.respond", |
| 326 | providerSessionId: oldSessionId, |
| 327 | requestId: RuntimeRequestId.make("runtime-request:effect-worker-interrupted"), |
| 328 | answers: { next: "Ship it" }, |
| 329 | }, |
| 330 | }, |
| 331 | { willRetry: false }, |
| 332 | ), |
| 333 | ); |
| 334 | }).pipe(Effect.provide(layer)); |
| 335 | assert.isTrue(Exit.hasInterrupts(exit)); |
| 336 | assert.equal(yield* Ref.get(reported), 0); |
| 337 | }), |
| 338 | ); |
| 339 | |
| 340 | it.effect("requeues a claim when a pre-execution worker check fails", () => |
| 341 | Effect.gen(function* () { |
| 342 | const now = DateTime.formatIso(yield* DateTime.now); |
| 343 | const effectId = "effect:worker-pre-execution-failure"; |
| 344 | const workerId = "worker-pre-execution-failure"; |
| 345 | const claimedEffect: EffectOutbox.OrchestrationEffectV2 = { |
| 346 | id: effectId, |
| 347 | commandId: CommandId.make("command:worker-pre-execution-failure"), |
| 348 | threadId: ThreadId.make("thread:worker-pre-execution-failure"), |
| 349 | request: { type: "terminal.cleanup" }, |
| 350 | status: "running", |
| 351 | attemptCount: 1, |
| 352 | availableAt: now, |
| 353 | leaseOwner: workerId, |
| 354 | createdAt: now, |
| 355 | updatedAt: now, |
| 356 | completedAt: null, |
| 357 | lastError: null, |
| 358 | }; |
| 359 | const retries = yield* Ref.make< |
| 360 | ReadonlyArray<{ |
| 361 | readonly effectId: string; |
| 362 | readonly workerId: string; |
| 363 | readonly error: string; |
| 364 | readonly delayMs: number; |
| 365 | }> |
| 366 | >([]); |
| 367 | const executionCount = yield* Ref.make(0); |
| 368 | const layerOutbox = Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 369 | claimNext: () => Effect.succeedSome(claimedEffect), |
| 370 | get: () => |
| 371 | Effect.fail( |
| 372 | new EffectOutbox.EffectOutboxError({ |
| 373 | operation: "get", |
| 374 | effectId, |
| 375 | cause: "simulated cancellation-state read failure", |
| 376 | }), |
| 377 | ), |
| 378 | retry: (input) => |
| 379 | Ref.update(retries, (existing) => [...existing, input]).pipe(Effect.as(true)), |
| 380 | }); |
| 381 | const layerExecutor = Layer.succeed( |
| 382 | EffectWorker.OrchestrationEffectExecutorV2, |
| 383 | EffectWorker.OrchestrationEffectExecutorV2.of({ |
| 384 | execute: () => Ref.update(executionCount, (count) => count + 1), |
| 385 | }), |
| 386 | ); |
| 387 | const layerWorker = EffectWorker.layerWithOptions({ workerId }).pipe( |
| 388 | Layer.provide(Layer.merge(layerOutbox, layerExecutor)), |
| 389 | ); |
| 390 | |
| 391 | const exit = yield* EffectWorker.OrchestrationEffectWorkerV2.pipe( |
| 392 | Effect.flatMap((worker) => worker.runOnce()), |
| 393 | Effect.provide(layerWorker), |
| 394 | Effect.exit, |
| 395 | ); |
| 396 | |
| 397 | assert.isTrue(Exit.isFailure(exit)); |
| 398 | if (Exit.isFailure(exit)) { |
| 399 | assert.include(Cause.pretty(exit.cause), "simulated cancellation-state read failure"); |
| 400 | } |
| 401 | assert.equal(yield* Ref.get(executionCount), 0); |
| 402 | const retry = (yield* Ref.get(retries))[0]; |
| 403 | assert.isDefined(retry); |
| 404 | assert.equal(retry.effectId, effectId); |
| 405 | assert.equal(retry.workerId, workerId); |
| 406 | assert.equal(retry.delayMs, 0); |
| 407 | assert.include(retry.error, "simulated cancellation-state read failure"); |
| 408 | }), |
| 409 | ); |
| 410 | |
| 411 | it.effect("keeps requeueing a failed claim while the database is locked", () => |
| 412 | Effect.gen(function* () { |
| 413 | const now = DateTime.formatIso(yield* DateTime.now); |
| 414 | const effectId = "effect:worker-requeue-lock-timeout"; |
| 415 | const workerId = "worker-requeue-lock-timeout"; |
| 416 | const claimedEffect: EffectOutbox.OrchestrationEffectV2 = { |
| 417 | id: effectId, |
| 418 | commandId: CommandId.make("command:worker-requeue-lock-timeout"), |
| 419 | threadId, |
| 420 | request: { type: "provider-runtime.continue", sourceRunId: runId }, |
| 421 | status: "running", |
| 422 | attemptCount: 1, |
| 423 | availableAt: now, |
| 424 | leaseOwner: workerId, |
| 425 | createdAt: now, |
| 426 | updatedAt: now, |
| 427 | completedAt: null, |
| 428 | lastError: null, |
| 429 | }; |
| 430 | const lockedWrites = yield* Ref.make(3); |
| 431 | const requeued = yield* Ref.make(false); |
| 432 | const firstLockTimeout = yield* Deferred.make<void>(); |
| 433 | const layerOutbox = Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 434 | claimNext: () => Effect.succeedSome(claimedEffect), |
| 435 | get: () => Effect.succeedSome(claimedEffect), |
| 436 | awaitCancellation: () => Effect.never, |
| 437 | clearCancellation: () => Effect.void, |
| 438 | retry: () => |
| 439 | Effect.gen(function* () { |
| 440 | if ((yield* Ref.getAndUpdate(lockedWrites, (count) => count - 1)) > 0) { |
| 441 | yield* Deferred.succeed(firstLockTimeout, undefined); |
| 442 | return yield* new EffectOutbox.EffectOutboxError({ |
| 443 | operation: "retry", |
| 444 | effectId, |
| 445 | cause: new SqlError.SqlError({ |
| 446 | reason: new SqlError.LockTimeoutError({ cause: null }), |
| 447 | }), |
| 448 | }); |
| 449 | } |
| 450 | yield* Ref.set(requeued, true); |
| 451 | return true; |
| 452 | }), |
| 453 | }); |
| 454 | const layerExecutor = Layer.succeed( |
| 455 | EffectWorker.OrchestrationEffectExecutorV2, |
| 456 | EffectWorker.OrchestrationEffectExecutorV2.of({ |
| 457 | execute: () => |
| 458 | Effect.fail( |
| 459 | new EffectWorker.OrchestrationEffectExecutionError({ |
| 460 | effectId, |
| 461 | effectType: "provider-runtime.continue", |
| 462 | cause: "simulated database is locked", |
| 463 | }), |
| 464 | ), |
| 465 | }), |
| 466 | ); |
| 467 | const layerWorker = EffectWorker.layerWithOptions({ workerId }).pipe( |
| 468 | Layer.provide(Layer.merge(layerOutbox, layerExecutor)), |
| 469 | ); |
| 470 | |
| 471 | const workerFiber = yield* EffectWorker.OrchestrationEffectWorkerV2.pipe( |
| 472 | Effect.flatMap((worker) => worker.runOnce()), |
| 473 | Effect.provide(layerWorker), |
| 474 | Effect.forkChild, |
| 475 | ); |
| 476 | yield* Deferred.await(firstLockTimeout); |
| 477 | yield* TestClock.adjust(Duration.millis(99)); |
| 478 | assert.isFalse(yield* Ref.get(requeued)); |
| 479 | yield* TestClock.adjust(Duration.seconds(1)); |
| 480 | yield* Effect.exit(Fiber.join(workerFiber)); |
| 481 | |
| 482 | assert.isTrue(yield* Ref.get(requeued)); |
| 483 | }), |
| 484 | ); |
| 485 | |
| 486 | it.effect.each([ |
| 487 | { name: "requeue", request: { type: "provider-runtime.continue", sourceRunId: runId } }, |
| 488 | { name: "terminalize", request: { type: "provider-turn.start", runId } }, |
| 489 | ] as const)("interrupts $name settlement retries when the worker shuts down", ({ request }) => |
| 490 | Effect.gen(function* () { |
| 491 | const claimedEffect: EffectOutbox.OrchestrationEffectV2 = { |
| 492 | ...restartEffect(yield* DateTime.now, { type: "detach" }), |
| 493 | request, |
| 494 | }; |
| 495 | const settlementStarted = yield* Deferred.make<void>(); |
| 496 | const releaseLock = yield* Ref.make(false); |
| 497 | const lockedWrite = Effect.gen(function* () { |
| 498 | yield* Deferred.succeed(settlementStarted, undefined); |
| 499 | // An in-flight database write can finish its own wait before noticing shutdown. |
| 500 | yield* Effect.sleep(Duration.millis(1)); |
| 501 | if (yield* Ref.get(releaseLock)) return true; |
| 502 | return yield* new EffectOutbox.EffectOutboxError({ |
| 503 | operation: "settlement", |
| 504 | effectId: claimedEffect.id, |
| 505 | cause: new SqlError.SqlError({ reason: new SqlError.LockTimeoutError({ cause: null }) }), |
| 506 | }); |
| 507 | }); |
| 508 | const layerOutbox = Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 509 | claimNext: () => Effect.succeedSome(claimedEffect), |
| 510 | get: () => Effect.succeedSome(claimedEffect), |
| 511 | awaitCancellation: () => Effect.never, |
| 512 | clearCancellation: () => Effect.void, |
| 513 | succeed: () => Effect.fail(new EffectOutbox.EffectOutboxError({ operation: "succeed" })), |
| 514 | retry: () => lockedWrite, |
| 515 | fail: () => lockedWrite, |
| 516 | }); |
| 517 | const layerExecutor = Layer.succeed( |
| 518 | EffectWorker.OrchestrationEffectExecutorV2, |
| 519 | EffectWorker.OrchestrationEffectExecutorV2.of({ execute: () => Effect.void }), |
| 520 | ); |
| 521 | const layerWorker = EffectWorker.layerWithOptions({ workerId: "test-worker" }).pipe( |
| 522 | Layer.provide(Layer.merge(layerOutbox, layerExecutor)), |
| 523 | ); |
| 524 | |
| 525 | const daemon = yield* EffectWorker.runDaemonWithOptions({ concurrency: 1 }).pipe( |
| 526 | Effect.provide(layerWorker), |
| 527 | Effect.forkChild, |
| 528 | ); |
| 529 | yield* Deferred.await(settlementStarted); |
| 530 | const shutdown = yield* Fiber.interrupt(daemon).pipe( |
| 531 | Effect.timeoutOption(Duration.millis(100)), |
| 532 | Effect.forkChild, |
| 533 | ); |
| 534 | yield* TestClock.adjust(Duration.millis(100)); |
| 535 | const stopped = yield* Fiber.join(shutdown); |
| 536 | yield* Ref.set(releaseLock, true); |
| 537 | yield* TestClock.adjust(Duration.seconds(1)); |
| 538 | yield* Fiber.await(daemon); |
| 539 | assert.isTrue(Option.isSome(stopped)); |
| 540 | }), |
| 541 | ); |
| 542 | |
| 543 | it.effect("arms cancellation before the durable pre-execution check", () => |
| 544 | Effect.gen(function* () { |
| 545 | const now = DateTime.formatIso(yield* DateTime.now); |
| 546 | const effectId = "effect:worker-cancellation-registration-race"; |
| 547 | const workerId = "worker-cancellation-registration-race"; |
| 548 | const claimedEffect: EffectOutbox.OrchestrationEffectV2 = { |
| 549 | id: effectId, |
| 550 | commandId: CommandId.make("command:worker-cancellation-registration-race"), |
| 551 | threadId: ThreadId.make("thread:worker-cancellation-registration-race"), |
| 552 | request: { type: "terminal.cleanup" }, |
| 553 | status: "running", |
| 554 | attemptCount: 1, |
| 555 | availableAt: now, |
| 556 | leaseOwner: workerId, |
| 557 | createdAt: now, |
| 558 | updatedAt: now, |
| 559 | completedAt: null, |
| 560 | lastError: null, |
| 561 | }; |
| 562 | const signal = yield* Deferred.make<void>(); |
| 563 | let cancellationArmed = false; |
| 564 | const executionCount = yield* Ref.make(0); |
| 565 | const settlementCount = yield* Ref.make(0); |
| 566 | const layerOutbox = Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 567 | claimNext: () => Effect.succeedSome(claimedEffect), |
| 568 | awaitCancellation: () => { |
| 569 | cancellationArmed = true; |
| 570 | return Deferred.await(signal); |
| 571 | }, |
| 572 | get: () => |
| 573 | Effect.gen(function* () { |
| 574 | if (cancellationArmed) { |
| 575 | yield* Deferred.succeed(signal, undefined); |
| 576 | } |
| 577 | return Option.some(claimedEffect); |
| 578 | }), |
| 579 | clearCancellation: () => Effect.void, |
| 580 | succeed: () => Ref.update(settlementCount, (count) => count + 1).pipe(Effect.as(true)), |
| 581 | }); |
| 582 | const layerExecutor = Layer.succeed( |
| 583 | EffectWorker.OrchestrationEffectExecutorV2, |
| 584 | EffectWorker.OrchestrationEffectExecutorV2.of({ |
| 585 | execute: () => |
| 586 | Effect.yieldNow.pipe(Effect.andThen(Ref.update(executionCount, (count) => count + 1))), |
| 587 | }), |
| 588 | ); |
| 589 | const layerWorker = EffectWorker.layerWithOptions({ workerId }).pipe( |
| 590 | Layer.provide(Layer.merge(layerOutbox, layerExecutor)), |
| 591 | ); |
| 592 | |
| 593 | const exit = yield* EffectWorker.OrchestrationEffectWorkerV2.pipe( |
| 594 | Effect.flatMap((worker) => worker.runOnce()), |
| 595 | Effect.provide(layerWorker), |
| 596 | Effect.exit, |
| 597 | ); |
| 598 | |
| 599 | if (Exit.isFailure(exit)) { |
| 600 | assert.fail(Cause.pretty(exit.cause)); |
| 601 | } |
| 602 | assert.equal(yield* Ref.get(executionCount), 0); |
| 603 | assert.equal(yield* Ref.get(settlementCount), 0); |
| 604 | }), |
| 605 | ); |
| 606 | |
| 607 | it.effect("terminalizes a process-bound claim when success settlement fails", () => |
| 608 | Effect.gen(function* () { |
| 609 | const now = DateTime.formatIso(yield* DateTime.now); |
| 610 | const effectId = "effect:worker-process-bound-settlement-failure"; |
| 611 | const workerId = "worker-process-bound-settlement-failure"; |
| 612 | const claimedEffect: EffectOutbox.OrchestrationEffectV2 = { |
| 613 | id: effectId, |
| 614 | commandId: CommandId.make("command:worker-process-bound-settlement-failure"), |
| 615 | threadId: ThreadId.make("thread:worker-process-bound-settlement-failure"), |
| 616 | request: { |
| 617 | type: "provider-turn.start", |
| 618 | runId: RunId.make("run:worker-process-bound-settlement-failure"), |
| 619 | }, |
| 620 | status: "running", |
| 621 | attemptCount: 1, |
| 622 | availableAt: now, |
| 623 | leaseOwner: workerId, |
| 624 | createdAt: now, |
| 625 | updatedAt: now, |
| 626 | completedAt: null, |
| 627 | lastError: null, |
| 628 | }; |
| 629 | const retries = yield* Ref.make(0); |
| 630 | const terminalErrors = yield* Ref.make<ReadonlyArray<string>>([]); |
| 631 | const executionCount = yield* Ref.make(0); |
| 632 | const layerOutbox = Layer.mock(EffectOutbox.EffectOutboxV2)({ |
| 633 | claimNext: () => Effect.succeedSome(claimedEffect), |
| 634 | get: () => Effect.succeedSome(claimedEffect), |
| 635 | awaitCancellation: () => Effect.never, |
| 636 | clearCancellation: () => Effect.void, |
| 637 | succeed: () => |
| 638 | Effect.fail( |
| 639 | new EffectOutbox.EffectOutboxError({ |
| 640 | operation: "succeed", |
| 641 | effectId, |
| 642 | cause: "simulated success settlement failure", |
| 643 | }), |
| 644 | ), |
| 645 | retry: () => Ref.update(retries, (count) => count + 1).pipe(Effect.as(true)), |
| 646 | fail: ({ error }) => |
| 647 | Ref.update(terminalErrors, (existing) => [...existing, error]).pipe(Effect.as(true)), |
| 648 | }); |
| 649 | const layerExecutor = Layer.succeed( |
| 650 | EffectWorker.OrchestrationEffectExecutorV2, |
| 651 | EffectWorker.OrchestrationEffectExecutorV2.of({ |
| 652 | execute: () => Ref.update(executionCount, (count) => count + 1), |
| 653 | }), |
| 654 | ); |
| 655 | const layerWorker = EffectWorker.layerWithOptions({ workerId }).pipe( |
| 656 | Layer.provide(Layer.merge(layerOutbox, layerExecutor)), |
| 657 | ); |
| 658 | |
| 659 | const exit = yield* EffectWorker.OrchestrationEffectWorkerV2.pipe( |
| 660 | Effect.flatMap((worker) => worker.runOnce()), |
| 661 | Effect.provide(layerWorker), |
| 662 | Effect.exit, |
| 663 | ); |
| 664 | |
| 665 | assert.isTrue(Exit.isFailure(exit)); |
| 666 | assert.equal(yield* Ref.get(executionCount), 1); |
| 667 | assert.equal(yield* Ref.get(retries), 0); |
| 668 | const terminalError = (yield* Ref.get(terminalErrors))[0]; |
| 669 | assert.isDefined(terminalError); |
| 670 | assert.include(terminalError, "after execution started"); |
| 671 | assert.include(terminalError, "simulated success settlement failure"); |
| 672 | }), |
| 673 | ); |
| 674 | |
| 675 | it.effect("settles a delegated child once its restart continuation fails for good", () => |
| 676 | Effect.gen(function* () { |
| 677 | const timestamp = DateTime.formatIso(yield* DateTime.now); |
| 678 | const events = yield* Ref.make<ReadonlyArray<string>>([]); |
| 679 | const recovered = yield* Ref.make<ReadonlyArray<ThreadId>>([]); |
| 680 | const layer = layerExecutorFor({ |
| 681 | events, |
| 682 | continueAfterRestart: true, |
| 683 | threads: { |
| 684 | getThreadRecords: () => Effect.fail(new Error("provider instance removed") as never), |
| 685 | recoverDelegatedTask: (childThreadId) => |
| 686 | Ref.update(recovered, (ids) => [...ids, childThreadId]), |
| 687 | }, |
| 688 | }); |
| 689 | const effect: EffectOutbox.OrchestrationEffectV2 = { |
| 690 | id: `effect:restart-continuation:${runId}`, |
| 691 | commandId: CommandId.make("command:restart-continuation-failure"), |
| 692 | threadId, |
| 693 | request: { type: "provider-runtime.continue", sourceRunId: runId }, |
| 694 | status: "running", |
| 695 | attemptCount: 1, |
| 696 | availableAt: timestamp, |
| 697 | leaseOwner: "test-worker", |
| 698 | createdAt: timestamp, |
| 699 | updatedAt: timestamp, |
| 700 | completedAt: null, |
| 701 | lastError: null, |
| 702 | }; |
| 703 | yield* Effect.gen(function* () { |
| 704 | const executor = yield* EffectWorker.OrchestrationEffectExecutorV2; |
| 705 | assert.isTrue( |
| 706 | Exit.isFailure(yield* Effect.exit(executor.execute(effect, { willRetry: true }))), |
| 707 | ); |
| 708 | assert.deepEqual(yield* Ref.get(recovered), []); |
| 709 | assert.isTrue( |
| 710 | Exit.isFailure(yield* Effect.exit(executor.execute(effect, { willRetry: false }))), |
| 711 | ); |
| 712 | assert.deepEqual(yield* Ref.get(recovered), [threadId]); |
| 713 | }).pipe(Effect.provide(layer)); |
| 714 | }), |
| 715 | ); |
| 716 | |
| 717 | it.effect.each(["confirmed", "unconfirmed", "failed"] as const)( |
| 718 | "a native subagent stop only repairs %s provider evidence", |
| 719 | (outcome) => |
| 720 | Effect.gen(function* () { |
| 721 | const now = yield* DateTime.now; |
| 722 | const events = yield* Ref.make<ReadonlyArray<string>>([]); |
| 723 | const layer = layerExecutorFor({ |
| 724 | events, |
| 725 | interrupt: (input) => { |
| 726 | assert.equal(input.subagent?.nativeTaskId, "native-task"); |
| 727 | return Ref.update(events, (current) => [...current, "stop-child"]).pipe( |
| 728 | Effect.andThen( |
| 729 | outcome === "failed" |
| 730 | ? new ProviderTurnControlService.ProviderTurnControlError({ |
| 731 | threadId, |
| 732 | providerTurnId, |
| 733 | operation: "interrupt", |
| 734 | cause: "SDK stop failed", |
| 735 | }) |
| 736 | : Effect.succeed( |
| 737 | outcome === "confirmed" |
| 738 | ? { |
| 739 | status: "cancelled" as const, |
| 740 | completedAt: now, |
| 741 | result: "Stopped by Claude", |
| 742 | } |
| 743 | : undefined, |
| 744 | ), |
| 745 | ), |
| 746 | ); |
| 747 | }, |
| 748 | threads: { |
| 749 | dispatch: (command) => { |
| 750 | assert.equal(command.type, "thread.background-work.settle"); |
| 751 | if (command.type === "thread.background-work.settle") { |
| 752 | assert.equal(command.subagent?.id, "subagent"); |
| 753 | assert.deepEqual(command.subagent?.outcome, { |
| 754 | status: "cancelled", |
| 755 | completedAt: now, |
| 756 | result: "Stopped by Claude", |
| 757 | }); |
| 758 | } |
| 759 | return Ref.update(events, (current) => [...current, "settle-child"]).pipe( |
| 760 | Effect.as({ sequence: 1, storedEvents: [] }), |
| 761 | ); |
| 762 | }, |
| 763 | }, |
| 764 | }); |
| 765 | const execution = yield* Effect.gen(function* () { |
| 766 | const executor = yield* EffectWorker.OrchestrationEffectExecutorV2; |
| 767 | yield* executor.execute({ |
| 768 | ...restartEffect(now, { type: "detach" }), |
| 769 | request: { |
| 770 | type: "provider-turn.interrupt", |
| 771 | providerSessionId: oldSessionId, |
| 772 | providerThreadId, |
| 773 | providerTurnId, |
| 774 | subagent: { id: NodeId.make("subagent"), nativeTaskId: "native-task" }, |
| 775 | }, |
| 776 | }); |
| 777 | }).pipe(Effect.provide(layer), Effect.exit); |
| 778 | assert.equal(Exit.isFailure(execution), outcome === "failed"); |
| 779 | assert.deepEqual( |
| 780 | yield* Ref.get(events), |
| 781 | outcome === "confirmed" ? ["stop-child", "settle-child"] : ["stop-child"], |
| 782 | ); |
| 783 | }), |
| 784 | ); |
| 785 | |
| 786 | it.effect.each([3, 4])( |
| 787 | "enforces process-loss count %s after 22 minutes through the executor", |
| 788 | (processLossCount) => |
| 789 | Effect.gen(function* () { |
| 790 | const now = yield* DateTime.now; |
| 791 | const instanceId = ProviderInstanceId.make("codex"); |
| 792 | const events = yield* Ref.make<ReadonlyArray<string>>([]); |
| 793 | const layer = layerExecutorFor({ |
| 794 | events, |
| 795 | continueAfterRestart: true, |
| 796 | threads: { |
| 797 | getThreadRecords: () => |
| 798 | Effect.succeed({ |
| 799 | thread: { |
| 800 | id: threadId, |
| 801 | projectId: ProjectId.make("project:executor-recovery"), |
| 802 | providerInstanceId: instanceId, |
| 803 | archivedAt: null, |
| 804 | deletedAt: null, |
| 805 | lineage: { parentThreadId: null, relationshipToParent: null }, |
| 806 | }, |
| 807 | runs: [ |
| 808 | { |
| 809 | id: runId, |
| 810 | providerInstanceId: instanceId, |
| 811 | status: "cancelled", |
| 812 | requestedAt: DateTime.subtract(now, { minutes: 22 }), |
| 813 | completedAt: now, |
| 814 | ordinal: 1, |
| 815 | }, |
| 816 | ], |
| 817 | messages: [], |
| 818 | providerTurns: [], |
| 819 | runtimeRequests: [], |
| 820 | turnItems: [], |
| 821 | attempts: [], |
| 822 | } as unknown as OrchestrationV2ThreadProjection), |
| 823 | dispatch: () => |
| 824 | Ref.update(events, (entries) => [...entries, "dispatch"]).pipe(Effect.as({} as never)), |
| 825 | recoverDelegatedTask: () => |
| 826 | Ref.update(events, (entries) => [...entries, "recover-child"]), |
| 827 | }, |
| 828 | }); |
| 829 | yield* Effect.gen(function* () { |
| 830 | const executor = yield* EffectWorker.OrchestrationEffectExecutorV2; |
| 831 | yield* executor.execute({ |
| 832 | ...restartEffect(now, { type: "detach" }), |
| 833 | request: { type: "provider-runtime.continue", sourceRunId: runId, processLossCount }, |
| 834 | }); |
| 835 | }).pipe(Effect.provide(layer)); |
| 836 | assert.deepEqual( |
| 837 | yield* Ref.get(events), |
| 838 | processLossCount > 3 ? ["recover-child"] : ["dispatch", "recover-child"], |
| 839 | ); |
| 840 | }), |
| 841 | ); |