@devries-cameron20 / t3code Mirror · read-only / commits / cbc38a1b0f
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 19:45 UTC

fix(server): one unresponsive provider no longer stalls every thread's effects (#35027)

Julius Marminge committed Nov 12, 2026 at 02:31 UTC cbc38a1b0f9a23f823a0e7e0189fcb0d88f38872
2 files changed +103 -1
apps/server/src/orchestration-v2/EffectWorker.ts
+11 -1
@@ -5,6 +5,7 @@ import * as DateTime from "effect/DateTime";
5 import * as Duration from "effect/Duration";
6 import * as Effect from "effect/Effect";
7 import * as Exit from "effect/Exit";
8 +import * as Fiber from "effect/Fiber";
9 import * as Layer from "effect/Layer";
10 import * as Metric from "effect/Metric";
11 import * as Option from "effect/Option";
@@ -943,11 +944,15 @@ export interface OrchestrationEffectDaemonOptions {
944
945 const DEFAULT_EFFECT_WORKER_CONCURRENCY = 4;
946 const DEFAULT_EFFECT_WORKER_LIVENESS_POLL_INTERVAL_MS = 30_000;
947 +// A provider request can go unanswered indefinitely. After this wait the slot claims other work
948 +// while the execution keeps its running row (which blocks its own thread) and settles it later.
949 +const EFFECT_WORKER_SLOT_WAIT = Duration.seconds(30);
950
951 export const runDaemonWithOptions = (options: OrchestrationEffectDaemonOptions = {}) =>
952 Effect.scoped(
953 Effect.gen(function* () {
954 const worker = yield* OrchestrationEffectWorkerV2;
955 + const scope = yield* Effect.scope;
956 const requestedConcurrency = options.concurrency ?? DEFAULT_EFFECT_WORKER_CONCURRENCY;
957 const concurrency = Number.isFinite(requestedConcurrency)
958 ? Math.max(1, Math.floor(requestedConcurrency))
@@ -962,13 +967,18 @@ export const runDaemonWithOptions = (options: OrchestrationEffectDaemonOptions =
967 // Notifications wake promptly; availableAt persists retry timing, and liveness polling recovers missed wakeups.
968 const runWorker = Effect.gen(function* () {
969 while (true) {
965 - const outcome = yield* worker.runOnce(claimLimits).pipe(
970 + const attempt = yield* worker.runOnce(claimLimits).pipe(
971 Effect.map((worked) => (worked ? ("worked" as const) : ("idle" as const))),
972 Effect.catchCause((cause) =>
973 Effect.logWarning("Orchestration effect worker failed", cause).pipe(
974 Effect.as("failed" as const),
975 ),
976 ),
977 + Effect.forkIn(scope),
978 + );
979 + const outcome = yield* Fiber.join(attempt).pipe(
980 + Effect.timeoutOption(EFFECT_WORKER_SLOT_WAIT),
981 + Effect.map(Option.getOrElse(() => "worked" as const)),
982 );
983 if (outcome === "worked") {
984 yield* Effect.yieldNow;
apps/server/src/orchestration-v2/FoundationPersistence.test.ts
+92
@@ -3254,6 +3254,98 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
3254 }),
3255 );
3256
3257 + it.effect(
3258 + "starts another thread's turn while every worker holds a start that never returns",
3259 + () =>
3260 + Effect.gen(function* () {
3261 + const outbox = yield* EffectOutbox.EffectOutboxV2;
3262 + const key = "foundation-hung-provider-starts";
3263 + const wedgedThreadIds = [0, 1].map((index) =>
3264 + ThreadId.make(`thread:${key}:wedged:${index}`),
3265 + );
3266 + const otherThreadId = ThreadId.make(`thread:${key}:other`);
3267 + const wedgedStarted = yield* Deferred.make<void>();
3268 + const otherStarted = yield* Deferred.make<void>();
3269 + const wedgedCount = yield* Ref.make(0);
3270 + yield* outbox.enqueue(
3271 + wedgedThreadIds.map((threadId, index) => ({
3272 + id: `effect:${key}:wedged:${index}`,
3273 + commandId: CommandId.make(`command:${key}:wedged:${index}`),
3274 + threadId,
3275 + request: {
3276 + type: "provider-turn.start",
3277 + runId: RunId.make(`run:${key}:wedged:${index}`),
3278 + },
3279 + })),
3280 + );
3281 +
3282 + const layerExecutor = Layer.succeed(
3283 + EffectWorker.OrchestrationEffectExecutorV2,
3284 + EffectWorker.OrchestrationEffectExecutorV2.of({
3285 + execute: (effect) =>
3286 + effect.threadId === otherThreadId
3287 + ? Deferred.succeed(otherStarted, undefined)
3288 + : Ref.updateAndGet(wedgedCount, (count) => count + 1).pipe(
3289 + Effect.flatMap((count) =>
3290 + count === wedgedThreadIds.length
3291 + ? Deferred.succeed(wedgedStarted, undefined)
3292 + : Effect.void,
3293 + ),
3294 + Effect.andThen(Effect.never),
3295 + ),
3296 + }),
3297 + );
3298 + const layerWorker = EffectWorker.layerWithOptions({
3299 + workerId: "hung-provider-starts-worker",
3300 + }).pipe(
3301 + Layer.provide(
3302 + Layer.merge(Layer.succeed(EffectOutbox.EffectOutboxV2, outbox), layerExecutor),
3303 + ),
3304 + );
3305 +
3306 + yield* Effect.gen(function* () {
3307 + yield* EffectWorker.runDaemonWithOptions({ concurrency: wedgedThreadIds.length }).pipe(
3308 + Effect.forkScoped,
3309 + );
3310 + yield* Deferred.await(wedgedStarted);
3311 + yield* outbox.enqueue([
3312 + {
3313 + id: `effect:${key}:other`,
3314 + commandId: CommandId.make(`command:${key}:other`),
3315 + threadId: otherThreadId,
3316 + request: { type: "provider-turn.start", runId: RunId.make(`run:${key}:other`) },
3317 + },
3318 + ]);
3319 + yield* outbox.notifyAvailable();
3320 +
3321 + const other = yield* Deferred.await(otherStarted).pipe(
3322 + Effect.timeoutOption("1 hour"),
3323 + Effect.forkChild,
3324 + );
3325 + yield* TestClock.adjust("1 hour");
3326 + assert.isTrue(
3327 + Option.isSome(yield* Fiber.join(other)),
3328 + "another thread's provider-turn.start never ran behind starts that never return",
3329 + );
3330 + }).pipe(
3331 + Effect.provide(layerWorker),
3332 + Effect.scoped,
3333 + Effect.ensuring(
3334 + Effect.forEach(
3335 + [...wedgedThreadIds, otherThreadId],
3336 + (threadId) =>
3337 + outbox.cancelUnsettled({
3338 + threadId,
3339 + effectTypes: ["provider-turn.start"],
3340 + reason: "fixture complete",
3341 + }),
3342 + { discard: true },
3343 + ).pipe(Effect.orDie),
3344 + ),
3345 + );
3346 + }),
3347 + );
3348 +
3349 it.effect(
3350 "persists shutdown continuation intent through the real event sink without domain events",
3351 () =>