@devries-cameron20 / t3code-1 / commits / 36d4d711b7
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): bound hung delegated stop effects so later runs can start (#25675)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 27, 2026 at 07:17 UTC 36d4d711b79c73eb33131944f73ea4a8c6156484
2 files changed +112 -1
apps/server/src/orchestration-v2/EffectWorker.ts
+24 -1
@@ -562,6 +562,29 @@ export const layerExecutor: Layer.Layer<
562 }),
563 );
564
565 +// A running effect blocks later work on its thread and its lease is never reclaimed in-process,
566 +// so a stop that hangs must give up and fall through to the normal retry/fail path.
567 +const DELEGATED_TASKS_STOP_TIMEOUT = Duration.seconds(30);
568 +
569 +const boundExecution =
570 + (effect: EffectOutbox.OrchestrationEffectV2) =>
571 + (execution: Effect.Effect<void, OrchestrationEffectExecutionError>) =>
572 + effect.request.type === "delegated-tasks.stop"
573 + ? execution.pipe(
574 + Effect.timeoutOrElse({
575 + duration: DELEGATED_TASKS_STOP_TIMEOUT,
576 + orElse: () =>
577 + Effect.fail(
578 + new OrchestrationEffectExecutionError({
579 + effectId: effect.id,
580 + effectType: effect.request.type,
581 + cause: `Timed out after ${Duration.format(DELEGATED_TASKS_STOP_TIMEOUT)}.`,
582 + }),
583 + ),
584 + }),
585 + )
586 + : execution;
587 +
588 export class OrchestrationEffectWorkerError extends Schema.TaggedError<OrchestrationEffectWorkerError>()(
589 "OrchestrationEffectWorkerError",
590 {
@@ -747,7 +770,7 @@ export const layerWithOptions = (
770
771 const execution = executor
772 .execute(effect, { willRetry: effect.attemptCount < maxAttempts })
750 - .pipe(Effect.as("executed" as const));
773 + .pipe(boundExecution(effect), Effect.as("executed" as const));
774 const exit = yield* Effect.exit(Effect.raceFirst(execution, cancellation)).pipe(
775 Effect.ensuring(outbox.clearCancellation(effect.id)),
776 );
apps/server/src/orchestration-v2/FoundationPersistence.test.ts
+88
@@ -2594,6 +2594,94 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
2594 }),
2595 );
2596
2597 + it.effect.each([{ replayed: false }, { replayed: true }] as const)(
2598 + "starts later thread work behind a delegated-task stop that never returns (replayed: $replayed)",
2599 + ({ replayed }) =>
2600 + Effect.gen(function* () {
2601 + const outbox = yield* EffectOutbox.EffectOutboxV2;
2602 + const key = `foundation-hung-delegated-stop:${replayed}`;
2603 + const threadId = ThreadId.make(`thread:${key}`);
2604 + const stopEffectId = `effect:${key}:stop`;
2605 + const stopStarted = yield* Deferred.make<void>();
2606 + const turnStarted = yield* Deferred.make<void>();
2607 + yield* outbox.enqueue([
2608 + {
2609 + id: stopEffectId,
2610 + commandId: CommandId.make(`command:${key}:stop`),
2611 + threadId,
2612 + request: { type: "delegated-tasks.stop" },
2613 + },
2614 + ]);
2615 + if (replayed) {
2616 + assert.isTrue(
2617 + Option.isSome(
2618 + yield* outbox.claimNext({ workerId: "crashed-worker", leaseDurationMs: 30_000 }),
2619 + ),
2620 + );
2621 + assert.equal((yield* outbox.reconcileAfterProcessLoss).requeued, 1);
2622 + }
2623 +
2624 + const layerExecutor = Layer.succeed(
2625 + EffectWorker.OrchestrationEffectExecutorV2,
2626 + EffectWorker.OrchestrationEffectExecutorV2.of({
2627 + execute: (effect) =>
2628 + effect.request.type === "delegated-tasks.stop"
2629 + ? Deferred.succeed(stopStarted, undefined).pipe(Effect.andThen(Effect.never))
2630 + : Deferred.succeed(turnStarted, undefined),
2631 + }),
2632 + );
2633 + const layerWorker = EffectWorker.layerWithOptions({
2634 + workerId: "hung-delegated-stop-worker",
2635 + }).pipe(
2636 + Layer.provide(
2637 + Layer.merge(Layer.succeed(EffectOutbox.EffectOutboxV2, outbox), layerExecutor),
2638 + ),
2639 + );
2640 +
2641 + yield* Effect.gen(function* () {
2642 + yield* EffectWorker.runDaemonWithOptions({ concurrency: 2 }).pipe(Effect.forkScoped);
2643 + yield* Deferred.await(stopStarted);
2644 + yield* outbox.enqueue([
2645 + {
2646 + id: `effect:${key}:turn`,
2647 + commandId: CommandId.make(`command:${key}:turn`),
2648 + threadId,
2649 + request: { type: "provider-turn.start", runId: RunId.make(`run:${key}`) },
2650 + },
2651 + ]);
2652 + yield* outbox.notifyAvailable();
2653 +
2654 + const turn = yield* Deferred.await(turnStarted).pipe(
2655 + Effect.timeoutOption("1 hour"),
2656 + Effect.forkChild,
2657 + );
2658 + yield* TestClock.adjust("1 hour");
2659 + assert.isTrue(
2660 + Option.isSome(yield* Fiber.join(turn)),
2661 + "provider-turn.start never ran behind the hung delegated-tasks.stop",
2662 + );
2663 + const stop = yield* outbox.get(stopEffectId);
2664 + assert.isTrue(Option.isSome(stop));
2665 + if (Option.isSome(stop)) {
2666 + assert.equal(stop.value.status, "failed");
2667 + assert.include(stop.value.lastError ?? "", "Timed out");
2668 + }
2669 + }).pipe(
2670 + Effect.provide(layerWorker),
2671 + Effect.scoped,
2672 + Effect.ensuring(
2673 + outbox
2674 + .cancelUnsettled({
2675 + threadId,
2676 + effectTypes: ["delegated-tasks.stop", "provider-turn.start"],
2677 + reason: "fixture complete",
2678 + })
2679 + .pipe(Effect.orDie),
2680 + ),
2681 + );
2682 + }),
2683 + );
2684 +
2685 it.effect(
2686 "persists shutdown continuation intent through the real event sink without domain events",
2687 () =>