The best way to control Claude Code, Codex and more from any computer or phone

main
ts 841 lines 31.1 KB
Raw
1 import { assert, it } from "@effect/vitest";
2 import {
3 CheckpointId,
4 CheckpointScopeId,
5 CommandId,
6 NodeId,
7 ProjectId,
8 ProviderInstanceId,
9 ProviderSessionId,
10 ProviderThreadId,
11 ProviderTurnId,
12 RunAttemptId,
13 RunId,
14 RuntimeRequestId,
15 ThreadId,
16 VcsProcessExitError,
17 type OrchestrationV2ThreadProjection,
18 } from "@t3tools/contracts";
19 import * as Cause from "effect/Cause";
20 import * as DateTime from "effect/DateTime";
21 import * as Deferred from "effect/Deferred";
22 import * as Duration from "effect/Duration";
23 import * as Effect from "effect/Effect";
24 import * as Exit from "effect/Exit";
25 import * as Fiber from "effect/Fiber";
26 import * as Layer from "effect/Layer";
27 import * as Option from "effect/Option";
28 import * as Ref from "effect/Ref";
29 import * as SqlError from "effect/sql/SqlError";
30 import * as Stream from "effect/Stream";
31 import * as TestClock from "effect/testing/TestClock";
32
33 import * as CheckpointRollbackService from "./CheckpointRollbackService.ts";
34 import * as CheckpointService from "./CheckpointService.ts";
35 import * as EffectOutbox from "./EffectOutbox.ts";
36 import * as EffectWorker from "./EffectWorker.ts";
37 import * as ProjectionStore from "./ProjectionStore.ts";
38 import * as RunFinalizationService from "./RunFinalizationService.ts";
39 import * as ProviderSessionManager from "./ProviderSessionManager.ts";
40 import * as ProviderTurnControlService from "./ProviderTurnControlService.ts";
41 import * as ProviderTurnStartService from "./ProviderTurnStartService.ts";
42 import * as RuntimeRequestService from "./RuntimeRequestService.ts";
43 import * as ThreadTitleRegenerationService from "./ThreadTitleRegenerationService.ts";
44 import * as ThreadManagementService from "./ThreadManagementService.ts";
45 import * as ServerSettings from "../serverSettings.ts";
46
47 const threadId = ThreadId.make("thread:effect-worker-restart");
48 const oldSessionId = ProviderSessionId.make("provider-session:effect-worker-restart:old");
49 const providerThreadId = ProviderThreadId.make("provider-thread:effect-worker-restart");
50 const providerTurnId = ProviderTurnId.make("provider-turn:effect-worker-restart");
51 const attemptId = RunAttemptId.make("run-attempt:effect-worker-restart");
52 const runId = RunId.make("run:effect-worker-restart");
53
54 function restartEffect(
55 now: DateTime.Utc,
56 sessionTransition: NonNullable<
57 Extract<
58 EffectOutbox.OrchestrationEffectV2["request"],
59 { readonly type: "provider-turn.restart" }
60 >["sessionTransition"]
61 >,
62 ): EffectOutbox.OrchestrationEffectV2 {
63 const timestamp = DateTime.formatIso(now);
64 return {
65 id: `effect:restart:${sessionTransition.type}`,
66 commandId: CommandId.make(`command:restart:${sessionTransition.type}`),
67 threadId,
68 request: {
69 type: "provider-turn.restart",
70 providerSessionId: oldSessionId,
71 providerThreadId,
72 providerTurnId,
73 interruptedAttemptId: attemptId,
74 runId,
75 sessionTransition,
76 },
77 status: "running",
78 attemptCount: 1,
79 availableAt: timestamp,
80 leaseOwner: "test-worker",
81 createdAt: timestamp,
82 updatedAt: timestamp,
83 completedAt: null,
84 lastError: null,
85 };
86 }
87
88 function layerExecutorFor(input: {
89 readonly events: Ref.Ref<ReadonlyArray<string>>;
90 readonly failFirstStart?: Ref.Ref<boolean>;
91 readonly threads?: Partial<ThreadManagementService.ThreadManagementService["Service"]>;
92 readonly continueAfterRestart?: boolean;
93 readonly interrupt?: ProviderTurnControlService.ProviderTurnControlServiceV2Shape["interrupt"];
94 readonly rollback?: CheckpointRollbackService.CheckpointRollbackServiceV2Shape["execute"];
95 readonly respond?: RuntimeRequestService.RuntimeRequestServiceV2["Service"]["respond"];
96 }) {
97 const record = (event: string) => Ref.update(input.events, (events) => [...events, event]);
98 const layerDependencies = Layer.mergeAll(
99 Layer.succeed(
100 ProviderTurnControlService.ProviderTurnControlServiceV2,
101 ProviderTurnControlService.ProviderTurnControlServiceV2.of({
102 interrupt: input.interrupt ?? (() => Effect.void),
103 steer: () => Effect.void,
104 applyModelSelection: () => Effect.void,
105 applyThreadTitle: () => Effect.void,
106 interruptAndAwaitTerminal: (request) =>
107 record(
108 request.replacementProviderSessionId === undefined
109 ? "interrupt"
110 : `interrupt:${request.replacementProviderSessionId}`,
111 ),
112 }),
113 ),
114 Layer.succeed(
115 ProviderSessionManager.ProviderSessionManagerV2,
116 ProviderSessionManager.ProviderSessionManagerV2.of({
117 shutdown: Effect.void,
118 open: () => Effect.die("unused open"),
119 get: () => Effect.succeedNone,
120 isLive: () => Effect.succeed(false),
121 close: () => Effect.void,
122 closeInstance: () => Effect.void,
123 release: () => record("release"),
124 detach: () => record("detach"),
125 otherWorkingThreads: () => Effect.succeed([]),
126 releaseUnlessBusy: () => Effect.succeed(true),
127 forgetLoadedProviderThread: () => Effect.void,
128 subscribePromptSuggestion: () => Stream.empty,
129 }),
130 ),
131 Layer.succeed(
132 ProviderTurnStartService.ProviderTurnStartServiceV2,
133 ProviderTurnStartService.ProviderTurnStartServiceV2.of({
134 failStartingRun: () => Effect.void,
135 start: () =>
136 Effect.gen(function* () {
137 yield* record("start");
138 if (
139 input.failFirstStart !== undefined &&
140 (yield* Ref.getAndSet(input.failFirstStart, false))
141 ) {
142 return yield* new ProviderTurnStartService.ProviderTurnStartError({
143 runId,
144 cause: "simulated first start failure",
145 });
146 }
147 }),
148 }),
149 ),
150 Layer.succeed(
151 RunFinalizationService.RunFinalizationService,
152 RunFinalizationService.RunFinalizationService.of({ finalize: () => Effect.void }),
153 ),
154 Layer.succeed(
155 CheckpointRollbackService.CheckpointRollbackServiceV2,
156 CheckpointRollbackService.CheckpointRollbackServiceV2.of({
157 execute: input.rollback ?? (() => Effect.void),
158 }),
159 ),
160 Layer.succeed(
161 RuntimeRequestService.RuntimeRequestServiceV2,
162 RuntimeRequestService.RuntimeRequestServiceV2.of({
163 respond: input.respond ?? (() => Effect.void),
164 }),
165 ),
166 Layer.succeed(
167 ThreadTitleRegenerationService.ThreadTitleRegenerationService,
168 ThreadTitleRegenerationService.ThreadTitleRegenerationService.of({
169 execute: () => Effect.void,
170 }),
171 ),
172 );
173 return EffectWorker.layerExecutor.pipe(
174 Layer.provide(
175 Layer.mergeAll(
176 layerDependencies,
177 Layer.mock(ThreadManagementService.ThreadManagementService)(input.threads ?? {}),
178 Layer.mock(ProjectionStore.ProjectionStoreV2)({}),
179 ServerSettings.layerTest(
180 input.continueAfterRestart === true ? { continueThreadsAfterServerUpdate: true } : {},
181 ),
182 ),
183 ),
184 );
185 }
186
187 it.effect.each([
188 {
189 name: "settles a stopped run when its adapter has already lost the native turn",
190 unresponsive: false,
191 dispatched: ["thread.background-work.settle"],
192 },
193 {
194 name: "leaves the run running and flags the Stop when the provider never confirms it",
195 unresponsive: true,
196 dispatched: ["thread.interrupt.unresponsive"],
197 },
198 ])("$name", ({ unresponsive, dispatched }) =>
199 Effect.gen(function* () {
200 const now = yield* DateTime.now;
201 const events = yield* Ref.make<ReadonlyArray<string>>([]);
202 const layer = layerExecutorFor({
203 events,
204 interrupt: () =>
205 new ProviderTurnControlService.ProviderTurnControlError({
206 threadId,
207 operation: "interrupt",
208 providerTurnId,
209 ...(unresponsive ? { unresponsive } : {}),
210 cause: unresponsive ? "Timed out." : "Provider turn is not active.",
211 }),
212 threads: {
213 dispatch: (command) =>
214 Ref.update(events, (current) => [...current, command.type]).pipe(
215 Effect.as({ sequence: 1, storedEvents: [] }),
216 ),
217 },
218 });
219 yield* Effect.gen(function* () {
220 const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
221 yield* executor.execute({
222 ...restartEffect(now, { type: "detach" }),
223 request: {
224 type: "provider-turn.interrupt",
225 providerSessionId: oldSessionId,
226 providerThreadId,
227 providerTurnId,
228 },
229 });
230 }).pipe(Effect.provide(layer));
231 assert.deepEqual(yield* Ref.get(events), dispatched);
232 }),
233 );
234
235 const lockedIndexMessage =
236 "Git's index is locked. Wait for other Git operations to finish, then try again.";
237 const rollbackCheckpointId = CheckpointId.make("checkpoint:effect-worker-rollback");
238 const rollbackScopeId = CheckpointScopeId.make("checkpoint-scope:effect-worker-rollback");
239 const restoreFailure = (detail: string) =>
240 new CheckpointRollbackService.CheckpointRollbackExecutionError({
241 reason: "unexpected-failure",
242 threadId,
243 providerThreadId,
244 checkpointId: rollbackCheckpointId,
245 cause: new CheckpointService.CheckpointRestoreError({
246 scopeId: rollbackScopeId,
247 checkpointId: rollbackCheckpointId,
248 cause: new VcsProcessExitError({
249 operation: "GitVcsDriverCore.restoreCheckpoint",
250 command: "git",
251 cwd: "/private/repo",
252 exitCode: 128,
253 detail,
254 failureKind: "command-failed",
255 }),
256 }),
257 });
258
259 it.effect.each([
260 {
261 name: "a held Git index lock",
262 failure: restoreFailure(lockedIndexMessage),
263 message: lockedIndexMessage,
264 },
265 {
266 name: "raw Git output",
267 failure: restoreFailure("error: could not clean /private/repo/secret.txt"),
268 message: CheckpointRollbackService.ROLLBACK_FAILED_MESSAGE,
269 },
270 ])("reports a rollback blocked by $name only after the last attempt", ({ failure, message }) =>
271 Effect.gen(function* () {
272 const now = yield* DateTime.now;
273 const failures = yield* Ref.make<ReadonlyArray<string>>([]);
274 const layer = layerExecutorFor({
275 events: yield* Ref.make<ReadonlyArray<string>>([]),
276 rollback: () => Effect.fail(failure),
277 threads: {
278 dispatch: (command) =>
279 Ref.update(failures, (current) =>
280 command.type === "checkpoint.rollback.fail" ? [...current, command.message] : current,
281 ).pipe(Effect.as({ sequence: 1, storedEvents: [] })),
282 },
283 });
284 yield* Effect.gen(function* () {
285 const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
286 const rollback: EffectOutbox.OrchestrationEffectV2 = {
287 ...restartEffect(now, { type: "detach" }),
288 request: {
289 type: "provider-thread.rollback",
290 providerThreadId,
291 checkpointId: rollbackCheckpointId,
292 scopeId: rollbackScopeId,
293 restoreFiles: true,
294 },
295 };
296 yield* Effect.flip(executor.execute(rollback, { willRetry: true }));
297 assert.deepEqual(yield* Ref.get(failures), []);
298 yield* Effect.flip(executor.execute(rollback, { willRetry: false }));
299 }).pipe(Effect.provide(layer));
300 assert.deepEqual(yield* Ref.get(failures), [message]);
301 }),
302 );
303
304 it.effect("does not report an answer whose last delivery attempt is interrupted", () =>
305 Effect.gen(function* () {
306 const now = yield* DateTime.now;
307 const reported = yield* Ref.make(0);
308 const layer = layerExecutorFor({
309 events: yield* Ref.make<ReadonlyArray<string>>([]),
310 respond: () => Effect.interrupt,
311 threads: {
312 dispatch: (command) =>
313 Ref.update(reported, (count) =>
314 command.type === "runtime-request.delivery.fail" ? count + 1 : count,
315 ).pipe(Effect.as({ sequence: 1, storedEvents: [] })),
316 },
317 });
318 const exit = yield* Effect.gen(function* () {
319 const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
320 return yield* Effect.exit(
321 executor.execute(
322 {
323 ...restartEffect(now, { type: "detach" }),
324 request: {
325 type: "runtime-request.respond",
326 providerSessionId: oldSessionId,
327 requestId: RuntimeRequestId.make("runtime-request:effect-worker-interrupted"),
328 answers: { next: "Ship it" },
329 },
330 },
331 { willRetry: false },
332 ),
333 );
334 }).pipe(Effect.provide(layer));
335 assert.isTrue(Exit.hasInterrupts(exit));
336 assert.equal(yield* Ref.get(reported), 0);
337 }),
338 );
339
340 it.effect("requeues a claim when a pre-execution worker check fails", () =>
341 Effect.gen(function* () {
342 const now = DateTime.formatIso(yield* DateTime.now);
343 const effectId = "effect:worker-pre-execution-failure";
344 const workerId = "worker-pre-execution-failure";
345 const claimedEffect: EffectOutbox.OrchestrationEffectV2 = {
346 id: effectId,
347 commandId: CommandId.make("command:worker-pre-execution-failure"),
348 threadId: ThreadId.make("thread:worker-pre-execution-failure"),
349 request: { type: "terminal.cleanup" },
350 status: "running",
351 attemptCount: 1,
352 availableAt: now,
353 leaseOwner: workerId,
354 createdAt: now,
355 updatedAt: now,
356 completedAt: null,
357 lastError: null,
358 };
359 const retries = yield* Ref.make<
360 ReadonlyArray<{
361 readonly effectId: string;
362 readonly workerId: string;
363 readonly error: string;
364 readonly delayMs: number;
365 }>
366 >([]);
367 const executionCount = yield* Ref.make(0);
368 const layerOutbox = Layer.mock(EffectOutbox.EffectOutboxV2)({
369 claimNext: () => Effect.succeedSome(claimedEffect),
370 get: () =>
371 Effect.fail(
372 new EffectOutbox.EffectOutboxError({
373 operation: "get",
374 effectId,
375 cause: "simulated cancellation-state read failure",
376 }),
377 ),
378 retry: (input) =>
379 Ref.update(retries, (existing) => [...existing, input]).pipe(Effect.as(true)),
380 });
381 const layerExecutor = Layer.succeed(
382 EffectWorker.OrchestrationEffectExecutorV2,
383 EffectWorker.OrchestrationEffectExecutorV2.of({
384 execute: () => Ref.update(executionCount, (count) => count + 1),
385 }),
386 );
387 const layerWorker = EffectWorker.layerWithOptions({ workerId }).pipe(
388 Layer.provide(Layer.merge(layerOutbox, layerExecutor)),
389 );
390
391 const exit = yield* EffectWorker.OrchestrationEffectWorkerV2.pipe(
392 Effect.flatMap((worker) => worker.runOnce()),
393 Effect.provide(layerWorker),
394 Effect.exit,
395 );
396
397 assert.isTrue(Exit.isFailure(exit));
398 if (Exit.isFailure(exit)) {
399 assert.include(Cause.pretty(exit.cause), "simulated cancellation-state read failure");
400 }
401 assert.equal(yield* Ref.get(executionCount), 0);
402 const retry = (yield* Ref.get(retries))[0];
403 assert.isDefined(retry);
404 assert.equal(retry.effectId, effectId);
405 assert.equal(retry.workerId, workerId);
406 assert.equal(retry.delayMs, 0);
407 assert.include(retry.error, "simulated cancellation-state read failure");
408 }),
409 );
410
411 it.effect("keeps requeueing a failed claim while the database is locked", () =>
412 Effect.gen(function* () {
413 const now = DateTime.formatIso(yield* DateTime.now);
414 const effectId = "effect:worker-requeue-lock-timeout";
415 const workerId = "worker-requeue-lock-timeout";
416 const claimedEffect: EffectOutbox.OrchestrationEffectV2 = {
417 id: effectId,
418 commandId: CommandId.make("command:worker-requeue-lock-timeout"),
419 threadId,
420 request: { type: "provider-runtime.continue", sourceRunId: runId },
421 status: "running",
422 attemptCount: 1,
423 availableAt: now,
424 leaseOwner: workerId,
425 createdAt: now,
426 updatedAt: now,
427 completedAt: null,
428 lastError: null,
429 };
430 const lockedWrites = yield* Ref.make(3);
431 const requeued = yield* Ref.make(false);
432 const firstLockTimeout = yield* Deferred.make<void>();
433 const layerOutbox = Layer.mock(EffectOutbox.EffectOutboxV2)({
434 claimNext: () => Effect.succeedSome(claimedEffect),
435 get: () => Effect.succeedSome(claimedEffect),
436 awaitCancellation: () => Effect.never,
437 clearCancellation: () => Effect.void,
438 retry: () =>
439 Effect.gen(function* () {
440 if ((yield* Ref.getAndUpdate(lockedWrites, (count) => count - 1)) > 0) {
441 yield* Deferred.succeed(firstLockTimeout, undefined);
442 return yield* new EffectOutbox.EffectOutboxError({
443 operation: "retry",
444 effectId,
445 cause: new SqlError.SqlError({
446 reason: new SqlError.LockTimeoutError({ cause: null }),
447 }),
448 });
449 }
450 yield* Ref.set(requeued, true);
451 return true;
452 }),
453 });
454 const layerExecutor = Layer.succeed(
455 EffectWorker.OrchestrationEffectExecutorV2,
456 EffectWorker.OrchestrationEffectExecutorV2.of({
457 execute: () =>
458 Effect.fail(
459 new EffectWorker.OrchestrationEffectExecutionError({
460 effectId,
461 effectType: "provider-runtime.continue",
462 cause: "simulated database is locked",
463 }),
464 ),
465 }),
466 );
467 const layerWorker = EffectWorker.layerWithOptions({ workerId }).pipe(
468 Layer.provide(Layer.merge(layerOutbox, layerExecutor)),
469 );
470
471 const workerFiber = yield* EffectWorker.OrchestrationEffectWorkerV2.pipe(
472 Effect.flatMap((worker) => worker.runOnce()),
473 Effect.provide(layerWorker),
474 Effect.forkChild,
475 );
476 yield* Deferred.await(firstLockTimeout);
477 yield* TestClock.adjust(Duration.millis(99));
478 assert.isFalse(yield* Ref.get(requeued));
479 yield* TestClock.adjust(Duration.seconds(1));
480 yield* Effect.exit(Fiber.join(workerFiber));
481
482 assert.isTrue(yield* Ref.get(requeued));
483 }),
484 );
485
486 it.effect.each([
487 { name: "requeue", request: { type: "provider-runtime.continue", sourceRunId: runId } },
488 { name: "terminalize", request: { type: "provider-turn.start", runId } },
489 ] as const)("interrupts $name settlement retries when the worker shuts down", ({ request }) =>
490 Effect.gen(function* () {
491 const claimedEffect: EffectOutbox.OrchestrationEffectV2 = {
492 ...restartEffect(yield* DateTime.now, { type: "detach" }),
493 request,
494 };
495 const settlementStarted = yield* Deferred.make<void>();
496 const releaseLock = yield* Ref.make(false);
497 const lockedWrite = Effect.gen(function* () {
498 yield* Deferred.succeed(settlementStarted, undefined);
499 // An in-flight database write can finish its own wait before noticing shutdown.
500 yield* Effect.sleep(Duration.millis(1));
501 if (yield* Ref.get(releaseLock)) return true;
502 return yield* new EffectOutbox.EffectOutboxError({
503 operation: "settlement",
504 effectId: claimedEffect.id,
505 cause: new SqlError.SqlError({ reason: new SqlError.LockTimeoutError({ cause: null }) }),
506 });
507 });
508 const layerOutbox = Layer.mock(EffectOutbox.EffectOutboxV2)({
509 claimNext: () => Effect.succeedSome(claimedEffect),
510 get: () => Effect.succeedSome(claimedEffect),
511 awaitCancellation: () => Effect.never,
512 clearCancellation: () => Effect.void,
513 succeed: () => Effect.fail(new EffectOutbox.EffectOutboxError({ operation: "succeed" })),
514 retry: () => lockedWrite,
515 fail: () => lockedWrite,
516 });
517 const layerExecutor = Layer.succeed(
518 EffectWorker.OrchestrationEffectExecutorV2,
519 EffectWorker.OrchestrationEffectExecutorV2.of({ execute: () => Effect.void }),
520 );
521 const layerWorker = EffectWorker.layerWithOptions({ workerId: "test-worker" }).pipe(
522 Layer.provide(Layer.merge(layerOutbox, layerExecutor)),
523 );
524
525 const daemon = yield* EffectWorker.runDaemonWithOptions({ concurrency: 1 }).pipe(
526 Effect.provide(layerWorker),
527 Effect.forkChild,
528 );
529 yield* Deferred.await(settlementStarted);
530 const shutdown = yield* Fiber.interrupt(daemon).pipe(
531 Effect.timeoutOption(Duration.millis(100)),
532 Effect.forkChild,
533 );
534 yield* TestClock.adjust(Duration.millis(100));
535 const stopped = yield* Fiber.join(shutdown);
536 yield* Ref.set(releaseLock, true);
537 yield* TestClock.adjust(Duration.seconds(1));
538 yield* Fiber.await(daemon);
539 assert.isTrue(Option.isSome(stopped));
540 }),
541 );
542
543 it.effect("arms cancellation before the durable pre-execution check", () =>
544 Effect.gen(function* () {
545 const now = DateTime.formatIso(yield* DateTime.now);
546 const effectId = "effect:worker-cancellation-registration-race";
547 const workerId = "worker-cancellation-registration-race";
548 const claimedEffect: EffectOutbox.OrchestrationEffectV2 = {
549 id: effectId,
550 commandId: CommandId.make("command:worker-cancellation-registration-race"),
551 threadId: ThreadId.make("thread:worker-cancellation-registration-race"),
552 request: { type: "terminal.cleanup" },
553 status: "running",
554 attemptCount: 1,
555 availableAt: now,
556 leaseOwner: workerId,
557 createdAt: now,
558 updatedAt: now,
559 completedAt: null,
560 lastError: null,
561 };
562 const signal = yield* Deferred.make<void>();
563 let cancellationArmed = false;
564 const executionCount = yield* Ref.make(0);
565 const settlementCount = yield* Ref.make(0);
566 const layerOutbox = Layer.mock(EffectOutbox.EffectOutboxV2)({
567 claimNext: () => Effect.succeedSome(claimedEffect),
568 awaitCancellation: () => {
569 cancellationArmed = true;
570 return Deferred.await(signal);
571 },
572 get: () =>
573 Effect.gen(function* () {
574 if (cancellationArmed) {
575 yield* Deferred.succeed(signal, undefined);
576 }
577 return Option.some(claimedEffect);
578 }),
579 clearCancellation: () => Effect.void,
580 succeed: () => Ref.update(settlementCount, (count) => count + 1).pipe(Effect.as(true)),
581 });
582 const layerExecutor = Layer.succeed(
583 EffectWorker.OrchestrationEffectExecutorV2,
584 EffectWorker.OrchestrationEffectExecutorV2.of({
585 execute: () =>
586 Effect.yieldNow.pipe(Effect.andThen(Ref.update(executionCount, (count) => count + 1))),
587 }),
588 );
589 const layerWorker = EffectWorker.layerWithOptions({ workerId }).pipe(
590 Layer.provide(Layer.merge(layerOutbox, layerExecutor)),
591 );
592
593 const exit = yield* EffectWorker.OrchestrationEffectWorkerV2.pipe(
594 Effect.flatMap((worker) => worker.runOnce()),
595 Effect.provide(layerWorker),
596 Effect.exit,
597 );
598
599 if (Exit.isFailure(exit)) {
600 assert.fail(Cause.pretty(exit.cause));
601 }
602 assert.equal(yield* Ref.get(executionCount), 0);
603 assert.equal(yield* Ref.get(settlementCount), 0);
604 }),
605 );
606
607 it.effect("terminalizes a process-bound claim when success settlement fails", () =>
608 Effect.gen(function* () {
609 const now = DateTime.formatIso(yield* DateTime.now);
610 const effectId = "effect:worker-process-bound-settlement-failure";
611 const workerId = "worker-process-bound-settlement-failure";
612 const claimedEffect: EffectOutbox.OrchestrationEffectV2 = {
613 id: effectId,
614 commandId: CommandId.make("command:worker-process-bound-settlement-failure"),
615 threadId: ThreadId.make("thread:worker-process-bound-settlement-failure"),
616 request: {
617 type: "provider-turn.start",
618 runId: RunId.make("run:worker-process-bound-settlement-failure"),
619 },
620 status: "running",
621 attemptCount: 1,
622 availableAt: now,
623 leaseOwner: workerId,
624 createdAt: now,
625 updatedAt: now,
626 completedAt: null,
627 lastError: null,
628 };
629 const retries = yield* Ref.make(0);
630 const terminalErrors = yield* Ref.make<ReadonlyArray<string>>([]);
631 const executionCount = yield* Ref.make(0);
632 const layerOutbox = Layer.mock(EffectOutbox.EffectOutboxV2)({
633 claimNext: () => Effect.succeedSome(claimedEffect),
634 get: () => Effect.succeedSome(claimedEffect),
635 awaitCancellation: () => Effect.never,
636 clearCancellation: () => Effect.void,
637 succeed: () =>
638 Effect.fail(
639 new EffectOutbox.EffectOutboxError({
640 operation: "succeed",
641 effectId,
642 cause: "simulated success settlement failure",
643 }),
644 ),
645 retry: () => Ref.update(retries, (count) => count + 1).pipe(Effect.as(true)),
646 fail: ({ error }) =>
647 Ref.update(terminalErrors, (existing) => [...existing, error]).pipe(Effect.as(true)),
648 });
649 const layerExecutor = Layer.succeed(
650 EffectWorker.OrchestrationEffectExecutorV2,
651 EffectWorker.OrchestrationEffectExecutorV2.of({
652 execute: () => Ref.update(executionCount, (count) => count + 1),
653 }),
654 );
655 const layerWorker = EffectWorker.layerWithOptions({ workerId }).pipe(
656 Layer.provide(Layer.merge(layerOutbox, layerExecutor)),
657 );
658
659 const exit = yield* EffectWorker.OrchestrationEffectWorkerV2.pipe(
660 Effect.flatMap((worker) => worker.runOnce()),
661 Effect.provide(layerWorker),
662 Effect.exit,
663 );
664
665 assert.isTrue(Exit.isFailure(exit));
666 assert.equal(yield* Ref.get(executionCount), 1);
667 assert.equal(yield* Ref.get(retries), 0);
668 const terminalError = (yield* Ref.get(terminalErrors))[0];
669 assert.isDefined(terminalError);
670 assert.include(terminalError, "after execution started");
671 assert.include(terminalError, "simulated success settlement failure");
672 }),
673 );
674
675 it.effect("settles a delegated child once its restart continuation fails for good", () =>
676 Effect.gen(function* () {
677 const timestamp = DateTime.formatIso(yield* DateTime.now);
678 const events = yield* Ref.make<ReadonlyArray<string>>([]);
679 const recovered = yield* Ref.make<ReadonlyArray<ThreadId>>([]);
680 const layer = layerExecutorFor({
681 events,
682 continueAfterRestart: true,
683 threads: {
684 getThreadRecords: () => Effect.fail(new Error("provider instance removed") as never),
685 recoverDelegatedTask: (childThreadId) =>
686 Ref.update(recovered, (ids) => [...ids, childThreadId]),
687 },
688 });
689 const effect: EffectOutbox.OrchestrationEffectV2 = {
690 id: `effect:restart-continuation:${runId}`,
691 commandId: CommandId.make("command:restart-continuation-failure"),
692 threadId,
693 request: { type: "provider-runtime.continue", sourceRunId: runId },
694 status: "running",
695 attemptCount: 1,
696 availableAt: timestamp,
697 leaseOwner: "test-worker",
698 createdAt: timestamp,
699 updatedAt: timestamp,
700 completedAt: null,
701 lastError: null,
702 };
703 yield* Effect.gen(function* () {
704 const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
705 assert.isTrue(
706 Exit.isFailure(yield* Effect.exit(executor.execute(effect, { willRetry: true }))),
707 );
708 assert.deepEqual(yield* Ref.get(recovered), []);
709 assert.isTrue(
710 Exit.isFailure(yield* Effect.exit(executor.execute(effect, { willRetry: false }))),
711 );
712 assert.deepEqual(yield* Ref.get(recovered), [threadId]);
713 }).pipe(Effect.provide(layer));
714 }),
715 );
716
717 it.effect.each(["confirmed", "unconfirmed", "failed"] as const)(
718 "a native subagent stop only repairs %s provider evidence",
719 (outcome) =>
720 Effect.gen(function* () {
721 const now = yield* DateTime.now;
722 const events = yield* Ref.make<ReadonlyArray<string>>([]);
723 const layer = layerExecutorFor({
724 events,
725 interrupt: (input) => {
726 assert.equal(input.subagent?.nativeTaskId, "native-task");
727 return Ref.update(events, (current) => [...current, "stop-child"]).pipe(
728 Effect.andThen(
729 outcome === "failed"
730 ? new ProviderTurnControlService.ProviderTurnControlError({
731 threadId,
732 providerTurnId,
733 operation: "interrupt",
734 cause: "SDK stop failed",
735 })
736 : Effect.succeed(
737 outcome === "confirmed"
738 ? {
739 status: "cancelled" as const,
740 completedAt: now,
741 result: "Stopped by Claude",
742 }
743 : undefined,
744 ),
745 ),
746 );
747 },
748 threads: {
749 dispatch: (command) => {
750 assert.equal(command.type, "thread.background-work.settle");
751 if (command.type === "thread.background-work.settle") {
752 assert.equal(command.subagent?.id, "subagent");
753 assert.deepEqual(command.subagent?.outcome, {
754 status: "cancelled",
755 completedAt: now,
756 result: "Stopped by Claude",
757 });
758 }
759 return Ref.update(events, (current) => [...current, "settle-child"]).pipe(
760 Effect.as({ sequence: 1, storedEvents: [] }),
761 );
762 },
763 },
764 });
765 const execution = yield* Effect.gen(function* () {
766 const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
767 yield* executor.execute({
768 ...restartEffect(now, { type: "detach" }),
769 request: {
770 type: "provider-turn.interrupt",
771 providerSessionId: oldSessionId,
772 providerThreadId,
773 providerTurnId,
774 subagent: { id: NodeId.make("subagent"), nativeTaskId: "native-task" },
775 },
776 });
777 }).pipe(Effect.provide(layer), Effect.exit);
778 assert.equal(Exit.isFailure(execution), outcome === "failed");
779 assert.deepEqual(
780 yield* Ref.get(events),
781 outcome === "confirmed" ? ["stop-child", "settle-child"] : ["stop-child"],
782 );
783 }),
784 );
785
786 it.effect.each([3, 4])(
787 "enforces process-loss count %s after 22 minutes through the executor",
788 (processLossCount) =>
789 Effect.gen(function* () {
790 const now = yield* DateTime.now;
791 const instanceId = ProviderInstanceId.make("codex");
792 const events = yield* Ref.make<ReadonlyArray<string>>([]);
793 const layer = layerExecutorFor({
794 events,
795 continueAfterRestart: true,
796 threads: {
797 getThreadRecords: () =>
798 Effect.succeed({
799 thread: {
800 id: threadId,
801 projectId: ProjectId.make("project:executor-recovery"),
802 providerInstanceId: instanceId,
803 archivedAt: null,
804 deletedAt: null,
805 lineage: { parentThreadId: null, relationshipToParent: null },
806 },
807 runs: [
808 {
809 id: runId,
810 providerInstanceId: instanceId,
811 status: "cancelled",
812 requestedAt: DateTime.subtract(now, { minutes: 22 }),
813 completedAt: now,
814 ordinal: 1,
815 },
816 ],
817 messages: [],
818 providerTurns: [],
819 runtimeRequests: [],
820 turnItems: [],
821 attempts: [],
822 } as unknown as OrchestrationV2ThreadProjection),
823 dispatch: () =>
824 Ref.update(events, (entries) => [...entries, "dispatch"]).pipe(Effect.as({} as never)),
825 recoverDelegatedTask: () =>
826 Ref.update(events, (entries) => [...entries, "recover-child"]),
827 },
828 });
829 yield* Effect.gen(function* () {
830 const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
831 yield* executor.execute({
832 ...restartEffect(now, { type: "detach" }),
833 request: { type: "provider-runtime.continue", sourceRunId: runId, processLossCount },
834 });
835 }).pipe(Effect.provide(layer));
836 assert.deepEqual(
837 yield* Ref.get(events),
838 processLossCount > 3 ? ["recover-child"] : ["dispatch", "recover-child"],
839 );
840 }),
841 );