The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 19:45 UTC
fix(server): one unresponsive provider no longer stalls every thread's effects (#35027)
Julius Marminge committed
Nov 12, 2026 at 02:31 UTC
cbc38a1b0f9a23f823a0e7e0189fcb0d88f38872
2 files changed
+103
-1
apps/server/src/orchestration-v2/EffectWorker.ts
+11
-1
@@ -5,6 +5,7 @@ import * as DateTime from "effect/DateTime";
5
import * as Duration from "effect/Duration";
6
import * as Effect from "effect/Effect";
7
import * as Exit from "effect/Exit";
8
+import * as Fiber from "effect/Fiber";
9
import * as Layer from "effect/Layer";
10
import * as Metric from "effect/Metric";
11
import * as Option from "effect/Option";
@@ -943,11 +944,15 @@ export interface OrchestrationEffectDaemonOptions {
944
945
const DEFAULT_EFFECT_WORKER_CONCURRENCY = 4;
946
const DEFAULT_EFFECT_WORKER_LIVENESS_POLL_INTERVAL_MS = 30_000;
947
+// A provider request can go unanswered indefinitely. After this wait the slot claims other work
948
+// while the execution keeps its running row (which blocks its own thread) and settles it later.
949
+const EFFECT_WORKER_SLOT_WAIT = Duration.seconds(30);
950
951
export const runDaemonWithOptions = (options: OrchestrationEffectDaemonOptions = {}) =>
952
Effect.scoped(
953
Effect.gen(function* () {
954
const worker = yield* OrchestrationEffectWorkerV2;
955
+ const scope = yield* Effect.scope;
956
const requestedConcurrency = options.concurrency ?? DEFAULT_EFFECT_WORKER_CONCURRENCY;
957
const concurrency = Number.isFinite(requestedConcurrency)
958
? Math.max(1, Math.floor(requestedConcurrency))
@@ -962,13 +967,18 @@ export const runDaemonWithOptions = (options: OrchestrationEffectDaemonOptions =
967
// Notifications wake promptly; availableAt persists retry timing, and liveness polling recovers missed wakeups.
968
const runWorker = Effect.gen(function* () {
969
while (true) {
965
- const outcome = yield* worker.runOnce(claimLimits).pipe(
970
+ const attempt = yield* worker.runOnce(claimLimits).pipe(
971
Effect.map((worked) => (worked ? ("worked" as const) : ("idle" as const))),
972
Effect.catchCause((cause) =>
973
Effect.logWarning("Orchestration effect worker failed", cause).pipe(
974
Effect.as("failed" as const),
975
),
976
),
977
+ Effect.forkIn(scope),
978
+ );
979
+ const outcome = yield* Fiber.join(attempt).pipe(
980
+ Effect.timeoutOption(EFFECT_WORKER_SLOT_WAIT),
981
+ Effect.map(Option.getOrElse(() => "worked" as const)),
982
);
983
if (outcome === "worked") {
984
yield* Effect.yieldNow;
apps/server/src/orchestration-v2/FoundationPersistence.test.ts
+92
@@ -3254,6 +3254,98 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
3254
}),
3255
);
3256
3257
+ it.effect(
3258
+ "starts another thread's turn while every worker holds a start that never returns",
3259
+ () =>
3260
+ Effect.gen(function* () {
3261
+ const outbox = yield* EffectOutbox.EffectOutboxV2;
3262
+ const key = "foundation-hung-provider-starts";
3263
+ const wedgedThreadIds = [0, 1].map((index) =>
3264
+ ThreadId.make(`thread:${key}:wedged:${index}`),
3265
+ );
3266
+ const otherThreadId = ThreadId.make(`thread:${key}:other`);
3267
+ const wedgedStarted = yield* Deferred.make<void>();
3268
+ const otherStarted = yield* Deferred.make<void>();
3269
+ const wedgedCount = yield* Ref.make(0);
3270
+ yield* outbox.enqueue(
3271
+ wedgedThreadIds.map((threadId, index) => ({
3272
+ id: `effect:${key}:wedged:${index}`,
3273
+ commandId: CommandId.make(`command:${key}:wedged:${index}`),
3274
+ threadId,
3275
+ request: {
3276
+ type: "provider-turn.start",
3277
+ runId: RunId.make(`run:${key}:wedged:${index}`),
3278
+ },
3279
+ })),
3280
+ );
3281
+
3282
+ const layerExecutor = Layer.succeed(
3283
+ EffectWorker.OrchestrationEffectExecutorV2,
3284
+ EffectWorker.OrchestrationEffectExecutorV2.of({
3285
+ execute: (effect) =>
3286
+ effect.threadId === otherThreadId
3287
+ ? Deferred.succeed(otherStarted, undefined)
3288
+ : Ref.updateAndGet(wedgedCount, (count) => count + 1).pipe(
3289
+ Effect.flatMap((count) =>
3290
+ count === wedgedThreadIds.length
3291
+ ? Deferred.succeed(wedgedStarted, undefined)
3292
+ : Effect.void,
3293
+ ),
3294
+ Effect.andThen(Effect.never),
3295
+ ),
3296
+ }),
3297
+ );
3298
+ const layerWorker = EffectWorker.layerWithOptions({
3299
+ workerId: "hung-provider-starts-worker",
3300
+ }).pipe(
3301
+ Layer.provide(
3302
+ Layer.merge(Layer.succeed(EffectOutbox.EffectOutboxV2, outbox), layerExecutor),
3303
+ ),
3304
+ );
3305
+
3306
+ yield* Effect.gen(function* () {
3307
+ yield* EffectWorker.runDaemonWithOptions({ concurrency: wedgedThreadIds.length }).pipe(
3308
+ Effect.forkScoped,
3309
+ );
3310
+ yield* Deferred.await(wedgedStarted);
3311
+ yield* outbox.enqueue([
3312
+ {
3313
+ id: `effect:${key}:other`,
3314
+ commandId: CommandId.make(`command:${key}:other`),
3315
+ threadId: otherThreadId,
3316
+ request: { type: "provider-turn.start", runId: RunId.make(`run:${key}:other`) },
3317
+ },
3318
+ ]);
3319
+ yield* outbox.notifyAvailable();
3320
+
3321
+ const other = yield* Deferred.await(otherStarted).pipe(
3322
+ Effect.timeoutOption("1 hour"),
3323
+ Effect.forkChild,
3324
+ );
3325
+ yield* TestClock.adjust("1 hour");
3326
+ assert.isTrue(
3327
+ Option.isSome(yield* Fiber.join(other)),
3328
+ "another thread's provider-turn.start never ran behind starts that never return",
3329
+ );
3330
+ }).pipe(
3331
+ Effect.provide(layerWorker),
3332
+ Effect.scoped,
3333
+ Effect.ensuring(
3334
+ Effect.forEach(
3335
+ [...wedgedThreadIds, otherThreadId],
3336
+ (threadId) =>
3337
+ outbox.cancelUnsettled({
3338
+ threadId,
3339
+ effectTypes: ["provider-turn.start"],
3340
+ reason: "fixture complete",
3341
+ }),
3342
+ { discard: true },
3343
+ ).pipe(Effect.orDie),
3344
+ ),
3345
+ );
3346
+ }),
3347
+ );
3348
+
3349
it.effect(
3350
"persists shutdown continuation intent through the real event sink without domain events",
3351
() =>