The best way to control Claude Code, Codex and more from any computer or phone
fix(server): bound hung delegated stop effects so later runs can start (#25675)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Oct 27, 2026 at 07:17 UTC
36d4d711b79c73eb33131944f73ea4a8c6156484
2 files changed
+112
-1
apps/server/src/orchestration-v2/EffectWorker.ts
+24
-1
@@ -562,6 +562,29 @@ export const layerExecutor: Layer.Layer<
562
}),
563
);
564
565
+// A running effect blocks later work on its thread and its lease is never reclaimed in-process,
566
+// so a stop that hangs must give up and fall through to the normal retry/fail path.
567
+const DELEGATED_TASKS_STOP_TIMEOUT = Duration.seconds(30);
568
+
569
+const boundExecution =
570
+ (effect: EffectOutbox.OrchestrationEffectV2) =>
571
+ (execution: Effect.Effect<void, OrchestrationEffectExecutionError>) =>
572
+ effect.request.type === "delegated-tasks.stop"
573
+ ? execution.pipe(
574
+ Effect.timeoutOrElse({
575
+ duration: DELEGATED_TASKS_STOP_TIMEOUT,
576
+ orElse: () =>
577
+ Effect.fail(
578
+ new OrchestrationEffectExecutionError({
579
+ effectId: effect.id,
580
+ effectType: effect.request.type,
581
+ cause: `Timed out after ${Duration.format(DELEGATED_TASKS_STOP_TIMEOUT)}.`,
582
+ }),
583
+ ),
584
+ }),
585
+ )
586
+ : execution;
587
+
588
export class OrchestrationEffectWorkerError extends Schema.TaggedError<OrchestrationEffectWorkerError>()(
589
"OrchestrationEffectWorkerError",
590
{
@@ -747,7 +770,7 @@ export const layerWithOptions = (
770
771
const execution = executor
772
.execute(effect, { willRetry: effect.attemptCount < maxAttempts })
750
- .pipe(Effect.as("executed" as const));
773
+ .pipe(boundExecution(effect), Effect.as("executed" as const));
774
const exit = yield* Effect.exit(Effect.raceFirst(execution, cancellation)).pipe(
775
Effect.ensuring(outbox.clearCancellation(effect.id)),
776
);
apps/server/src/orchestration-v2/FoundationPersistence.test.ts
+88
@@ -2594,6 +2594,94 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
2594
}),
2595
);
2596
2597
+ it.effect.each([{ replayed: false }, { replayed: true }] as const)(
2598
+ "starts later thread work behind a delegated-task stop that never returns (replayed: $replayed)",
2599
+ ({ replayed }) =>
2600
+ Effect.gen(function* () {
2601
+ const outbox = yield* EffectOutbox.EffectOutboxV2;
2602
+ const key = `foundation-hung-delegated-stop:${replayed}`;
2603
+ const threadId = ThreadId.make(`thread:${key}`);
2604
+ const stopEffectId = `effect:${key}:stop`;
2605
+ const stopStarted = yield* Deferred.make<void>();
2606
+ const turnStarted = yield* Deferred.make<void>();
2607
+ yield* outbox.enqueue([
2608
+ {
2609
+ id: stopEffectId,
2610
+ commandId: CommandId.make(`command:${key}:stop`),
2611
+ threadId,
2612
+ request: { type: "delegated-tasks.stop" },
2613
+ },
2614
+ ]);
2615
+ if (replayed) {
2616
+ assert.isTrue(
2617
+ Option.isSome(
2618
+ yield* outbox.claimNext({ workerId: "crashed-worker", leaseDurationMs: 30_000 }),
2619
+ ),
2620
+ );
2621
+ assert.equal((yield* outbox.reconcileAfterProcessLoss).requeued, 1);
2622
+ }
2623
+
2624
+ const layerExecutor = Layer.succeed(
2625
+ EffectWorker.OrchestrationEffectExecutorV2,
2626
+ EffectWorker.OrchestrationEffectExecutorV2.of({
2627
+ execute: (effect) =>
2628
+ effect.request.type === "delegated-tasks.stop"
2629
+ ? Deferred.succeed(stopStarted, undefined).pipe(Effect.andThen(Effect.never))
2630
+ : Deferred.succeed(turnStarted, undefined),
2631
+ }),
2632
+ );
2633
+ const layerWorker = EffectWorker.layerWithOptions({
2634
+ workerId: "hung-delegated-stop-worker",
2635
+ }).pipe(
2636
+ Layer.provide(
2637
+ Layer.merge(Layer.succeed(EffectOutbox.EffectOutboxV2, outbox), layerExecutor),
2638
+ ),
2639
+ );
2640
+
2641
+ yield* Effect.gen(function* () {
2642
+ yield* EffectWorker.runDaemonWithOptions({ concurrency: 2 }).pipe(Effect.forkScoped);
2643
+ yield* Deferred.await(stopStarted);
2644
+ yield* outbox.enqueue([
2645
+ {
2646
+ id: `effect:${key}:turn`,
2647
+ commandId: CommandId.make(`command:${key}:turn`),
2648
+ threadId,
2649
+ request: { type: "provider-turn.start", runId: RunId.make(`run:${key}`) },
2650
+ },
2651
+ ]);
2652
+ yield* outbox.notifyAvailable();
2653
+
2654
+ const turn = yield* Deferred.await(turnStarted).pipe(
2655
+ Effect.timeoutOption("1 hour"),
2656
+ Effect.forkChild,
2657
+ );
2658
+ yield* TestClock.adjust("1 hour");
2659
+ assert.isTrue(
2660
+ Option.isSome(yield* Fiber.join(turn)),
2661
+ "provider-turn.start never ran behind the hung delegated-tasks.stop",
2662
+ );
2663
+ const stop = yield* outbox.get(stopEffectId);
2664
+ assert.isTrue(Option.isSome(stop));
2665
+ if (Option.isSome(stop)) {
2666
+ assert.equal(stop.value.status, "failed");
2667
+ assert.include(stop.value.lastError ?? "", "Timed out");
2668
+ }
2669
+ }).pipe(
2670
+ Effect.provide(layerWorker),
2671
+ Effect.scoped,
2672
+ Effect.ensuring(
2673
+ outbox
2674
+ .cancelUnsettled({
2675
+ threadId,
2676
+ effectTypes: ["delegated-tasks.stop", "provider-turn.start"],
2677
+ reason: "fixture complete",
2678
+ })
2679
+ .pipe(Effect.orDie),
2680
+ ),
2681
+ );
2682
+ }),
2683
+ );
2684
+
2685
it.effect(
2686
"persists shutdown continuation intent through the real event sink without domain events",
2687
() =>