The best way to control Claude Code, Codex and more from any computer or phone
fix(server): a limited provider's queue stays held after another provider's turn succeeds (#22425)
Co-authored-by: Mantas Kantautas <mantas.kantautas@gmail.com> Co-authored-by: maria-rcks <maria@kuuro.net> Co-authored-by: Julius Marminge <julius0216@outlook.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Ibrahim Elkamali committed
Nov 11, 2026 at 12:53 UTC
8d236998d074799a6e88a495ba3112e87d6ab696
2 files changed
+165
-12
apps/server/src/orchestration-v2/Orchestrator.ts
+31
-12
@@ -899,9 +899,15 @@ function queueTailModelSelection(
899
);
900
}
901
902
+/**
903
+ * The provider's latest run stopped on its usage limit and nothing ran after it.
904
+ * With `now` (automatic starts), a run on another provider since does not lift
905
+ * a reported reset still in the future; an unknown reset is lifted, as transient 429s are.
906
+ */
907
function usageLimitHoldsQueue(
908
projection: Pick<OrchestrationV2ThreadProjection, "runs" | "turnItems" | "providerSessions">,
909
providerInstanceId: ModelSelection["instanceId"],
910
+ now?: DateTime.Utc,
911
): boolean {
912
let latestSession: OrchestrationV2ThreadProjection["providerSessions"][number] | undefined;
913
for (const session of projection.providerSessions) {
@@ -913,10 +919,16 @@ function usageLimitHoldsQueue(
919
latestSession = session;
920
}
921
}
916
- return (
917
- usageLimitBlockedRun(projection.runs, projection.turnItems, latestSession?.lastError ?? null)
918
- ?.providerInstanceId === providerInstanceId
922
+ const limited = usageLimitBlockedRun(
923
+ projection.runs.filter((run) => run.providerInstanceId === providerInstanceId),
924
+ projection.turnItems,
925
+ latestSession?.lastError ?? null,
926
);
927
+ if (limited === null) return false;
928
+ if (latestExecutedRun(projection.runs)?.id === limited.id) return true;
929
+ if (now === undefined) return false;
930
+ const resetAt = latestRootProviderFailure(limited, projection.turnItems)?.resetAt;
931
+ return resetAt != null && Date.parse(resetAt) > DateTime.toEpochMillis(now);
932
}
933
934
function latestStableRun(
@@ -1920,6 +1932,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1932
readonly failedRunId?: RunId;
1933
readonly endedRunId?: RunId;
1934
readonly queuedRunId?: RunId;
1935
+ readonly userStarted?: boolean;
1936
},
1937
) {
1938
if (!(yield* projectionStore.canStartQueuedRun(threadId))) return;
@@ -1951,7 +1964,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1964
providerInstanceId: modelSelection.instanceId,
1965
};
1966
// A limited provider would fail and consume every waiting message.
1954
- if (usageLimitHoldsQueue(projection, queuedRun.providerInstanceId)) {
1967
+ if (
1968
+ usageLimitHoldsQueue(
1969
+ projection,
1970
+ queuedRun.providerInstanceId,
1971
+ options?.userStarted === true ? undefined : yield* DateTime.now,
1972
+ )
1973
+ ) {
1974
return;
1975
}
1976
// Waits without a hold: lowering the thread's mode starts the queue again.
@@ -14009,10 +14028,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
14028
);
14029
if (command.type === "queue.resume" || command.type === "queued-run.start") {
14030
yield* mapDispatchError(command)(
14012
- startNextQueuedRun(
14013
- command.threadId,
14014
- command.type === "queued-run.start" ? { queuedRunId: command.runId } : undefined,
14015
- ),
14031
+ startNextQueuedRun(command.threadId, {
14032
+ userStarted: true,
14033
+ ...(command.type === "queued-run.start" ? { queuedRunId: command.runId } : {}),
14034
+ }),
14035
);
14036
}
14037
return {
@@ -14229,10 +14248,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
14248
(command.type === "thread.runtime-mode.set" && runtimeModeCeiling !== undefined)
14249
) {
14250
yield* mapDispatchError(command)(
14232
- startNextQueuedRun(
14233
- command.threadId,
14234
- command.type === "queued-run.start" ? { queuedRunId: command.runId } : undefined,
14235
- ),
14251
+ startNextQueuedRun(command.threadId, {
14252
+ userStarted: command.type !== "thread.runtime-mode.set",
14253
+ ...(command.type === "queued-run.start" ? { queuedRunId: command.runId } : {}),
14254
+ }),
14255
);
14256
}
14257
if (
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+134
@@ -6284,6 +6284,140 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
6284
}),
6285
);
6286
6287
+ it.effect.each(["the reset passes", "the user resumes"] as const)(
6288
+ "holds a limited provider's queue through another provider's turn until %s",
6289
+ (release) =>
6290
+ Effect.gen(function* () {
6291
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
6292
+ const eventSink = yield* EventSink.EventSinkV2;
6293
+ const threadId = ThreadId.make(`runtime-layer-queue-limit-switch-${release}`);
6294
+ const switchedSelection = { instanceId: alternateInstanceId, model: "gpt-5.5" };
6295
+ yield* orchestrator.dispatch({
6296
+ type: "thread.create",
6297
+ commandId: CommandId.make(`${threadId}:create`),
6298
+ threadId,
6299
+ createdBy: "user",
6300
+ creationSource: "web",
6301
+ projectId: ProjectId.make(`${threadId}:project`),
6302
+ title: "Limit switch",
6303
+ modelSelection,
6304
+ runtimeMode: "full-access",
6305
+ interactionMode: "default",
6306
+ branch: null,
6307
+ worktreePath: process.cwd(),
6308
+ });
6309
+ const send = (
6310
+ name: string,
6311
+ selection: typeof modelSelection | typeof switchedSelection,
6312
+ type: "start_immediately" | "queue_after_active",
6313
+ ) =>
6314
+ orchestrator.dispatch({
6315
+ type: "message.dispatch",
6316
+ commandId: CommandId.make(`${threadId}:${name}`),
6317
+ threadId,
6318
+ messageId: MessageId.make(`${threadId}:${name}`),
6319
+ text: name,
6320
+ attachments: [],
6321
+ createdBy: "user",
6322
+ creationSource: "web",
6323
+ modelSelection: selection,
6324
+ dispatchMode: { type },
6325
+ });
6326
+ yield* send("limited", modelSelection, "start_immediately");
6327
+ yield* send("queued", modelSelection, "queue_after_active");
6328
+ const [limited, queued] = (yield* orchestrator.getThreadProjection(threadId)).runs;
6329
+ assert.isDefined(limited);
6330
+ assert.isDefined(queued);
6331
+ const now = yield* DateTime.now;
6332
+ yield* eventSink.write({
6333
+ events: [
6334
+ {
6335
+ id: EventId.make(`${threadId}:error`),
6336
+ type: "turn-item.updated",
6337
+ threadId,
6338
+ runId: limited.id,
6339
+ nodeId: limited.rootNodeId!,
6340
+ providerInstanceId: limited.providerInstanceId,
6341
+ occurredAt: now,
6342
+ payload: {
6343
+ id: TurnItemId.make(`${threadId}:error`),
6344
+ type: "error",
6345
+ threadId,
6346
+ runId: limited.id,
6347
+ nodeId: limited.rootNodeId!,
6348
+ providerThreadId: limited.providerThreadId,
6349
+ providerTurnId: null,
6350
+ nativeItemRef: null,
6351
+ parentItemId: null,
6352
+ ordinal: 2,
6353
+ status: "failed",
6354
+ title: "Usage limit",
6355
+ startedAt: now,
6356
+ completedAt: now,
6357
+ updatedAt: now,
6358
+ failure: {
6359
+ class: "usage_limit",
6360
+ message: "Usage limit reached",
6361
+ code: "usage_limit",
6362
+ retryable: null,
6363
+ resetAt: DateTime.formatIso(DateTime.add(now, { hours: 1 })),
6364
+ },
6365
+ },
6366
+ },
6367
+ {
6368
+ id: EventId.make(`${threadId}:failed`),
6369
+ type: "run.updated",
6370
+ threadId,
6371
+ runId: limited.id,
6372
+ providerInstanceId: limited.providerInstanceId,
6373
+ occurredAt: now,
6374
+ payload: { ...limited, status: "failed", completedAt: now },
6375
+ },
6376
+ ],
6377
+ });
6378
+
6379
+ // A message on another provider starts ahead of the held queue and
6380
+ // completes there.
6381
+ yield* send("switch", switchedSelection, "start_immediately");
6382
+ const switched = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
6383
+ (run) => run.userMessageId === MessageId.make(`${threadId}:switch`),
6384
+ );
6385
+ assert.equal(switched?.status, "starting");
6386
+ yield* eventSink.write({
6387
+ events: [
6388
+ {
6389
+ id: EventId.make(`${threadId}:switch:completed`),
6390
+ type: "run.updated",
6391
+ threadId,
6392
+ runId: switched!.id,
6393
+ providerInstanceId: switched!.providerInstanceId,
6394
+ occurredAt: now,
6395
+ payload: { ...switched!, status: "completed", completedAt: now },
6396
+ },
6397
+ ],
6398
+ });
6399
+
6400
+ yield* orchestrator.resumeQueuedRuns;
6401
+ const held = yield* orchestrator.getThreadProjection(threadId);
6402
+ assert.equal(held.runs.find((run) => run.id === queued.id)?.status, "queued");
6403
+
6404
+ if (release === "the reset passes") {
6405
+ yield* TestClock.adjust("61 minutes");
6406
+ yield* orchestrator.resumeQueuedRuns;
6407
+ } else {
6408
+ yield* orchestrator.dispatch({
6409
+ type: "queue.resume",
6410
+ commandId: CommandId.make(`${threadId}:resume`),
6411
+ threadId,
6412
+ });
6413
+ }
6414
+ const reset = yield* orchestrator.getThreadProjection(threadId);
6415
+ const resumed = reset.runs.find((run) => run.id === queued.id);
6416
+ assert.equal(resumed?.status, "starting");
6417
+ assert.deepEqual(resumed?.modelSelection, modelSelection);
6418
+ }),
6419
+ );
6420
+
6421
it.effect.each([
6422
["server", modelSelection],
6423
["provider", modelSelection],