@devries-cameron20 / t3code-1 / commits / 8d236998d0
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): a limited provider's queue stays held after another provider's turn succeeds (#22425)

Co-authored-by: Mantas Kantautas <mantas.kantautas@gmail.com> Co-authored-by: maria-rcks <maria@kuuro.net> Co-authored-by: Julius Marminge <julius0216@outlook.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Ibrahim Elkamali committed Nov 11, 2026 at 12:53 UTC 8d236998d074799a6e88a495ba3112e87d6ab696
2 files changed +165 -12
apps/server/src/orchestration-v2/Orchestrator.ts
+31 -12
@@ -899,9 +899,15 @@ function queueTailModelSelection(
899 );
900 }
901
902 +/**
903 + * The provider's latest run stopped on its usage limit and nothing ran after it.
904 + * With `now` (automatic starts), a run on another provider since does not lift
905 + * a reported reset still in the future; an unknown reset is lifted, as transient 429s are.
906 + */
907 function usageLimitHoldsQueue(
908 projection: Pick<OrchestrationV2ThreadProjection, "runs" | "turnItems" | "providerSessions">,
909 providerInstanceId: ModelSelection["instanceId"],
910 + now?: DateTime.Utc,
911 ): boolean {
912 let latestSession: OrchestrationV2ThreadProjection["providerSessions"][number] | undefined;
913 for (const session of projection.providerSessions) {
@@ -913,10 +919,16 @@ function usageLimitHoldsQueue(
919 latestSession = session;
920 }
921 }
916 - return (
917 - usageLimitBlockedRun(projection.runs, projection.turnItems, latestSession?.lastError ?? null)
918 - ?.providerInstanceId === providerInstanceId
922 + const limited = usageLimitBlockedRun(
923 + projection.runs.filter((run) => run.providerInstanceId === providerInstanceId),
924 + projection.turnItems,
925 + latestSession?.lastError ?? null,
926 );
927 + if (limited === null) return false;
928 + if (latestExecutedRun(projection.runs)?.id === limited.id) return true;
929 + if (now === undefined) return false;
930 + const resetAt = latestRootProviderFailure(limited, projection.turnItems)?.resetAt;
931 + return resetAt != null && Date.parse(resetAt) > DateTime.toEpochMillis(now);
932 }
933
934 function latestStableRun(
@@ -1920,6 +1932,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1932 readonly failedRunId?: RunId;
1933 readonly endedRunId?: RunId;
1934 readonly queuedRunId?: RunId;
1935 + readonly userStarted?: boolean;
1936 },
1937 ) {
1938 if (!(yield* projectionStore.canStartQueuedRun(threadId))) return;
@@ -1951,7 +1964,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1964 providerInstanceId: modelSelection.instanceId,
1965 };
1966 // A limited provider would fail and consume every waiting message.
1954 - if (usageLimitHoldsQueue(projection, queuedRun.providerInstanceId)) {
1967 + if (
1968 + usageLimitHoldsQueue(
1969 + projection,
1970 + queuedRun.providerInstanceId,
1971 + options?.userStarted === true ? undefined : yield* DateTime.now,
1972 + )
1973 + ) {
1974 return;
1975 }
1976 // Waits without a hold: lowering the thread's mode starts the queue again.
@@ -14009,10 +14028,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
14028 );
14029 if (command.type === "queue.resume" || command.type === "queued-run.start") {
14030 yield* mapDispatchError(command)(
14012 - startNextQueuedRun(
14013 - command.threadId,
14014 - command.type === "queued-run.start" ? { queuedRunId: command.runId } : undefined,
14015 - ),
14031 + startNextQueuedRun(command.threadId, {
14032 + userStarted: true,
14033 + ...(command.type === "queued-run.start" ? { queuedRunId: command.runId } : {}),
14034 + }),
14035 );
14036 }
14037 return {
@@ -14229,10 +14248,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
14248 (command.type === "thread.runtime-mode.set" && runtimeModeCeiling !== undefined)
14249 ) {
14250 yield* mapDispatchError(command)(
14232 - startNextQueuedRun(
14233 - command.threadId,
14234 - command.type === "queued-run.start" ? { queuedRunId: command.runId } : undefined,
14235 - ),
14251 + startNextQueuedRun(command.threadId, {
14252 + userStarted: command.type !== "thread.runtime-mode.set",
14253 + ...(command.type === "queued-run.start" ? { queuedRunId: command.runId } : {}),
14254 + }),
14255 );
14256 }
14257 if (
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+134
@@ -6284,6 +6284,140 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
6284 }),
6285 );
6286
6287 + it.effect.each(["the reset passes", "the user resumes"] as const)(
6288 + "holds a limited provider's queue through another provider's turn until %s",
6289 + (release) =>
6290 + Effect.gen(function* () {
6291 + const orchestrator = yield* Orchestrator.OrchestratorV2;
6292 + const eventSink = yield* EventSink.EventSinkV2;
6293 + const threadId = ThreadId.make(`runtime-layer-queue-limit-switch-${release}`);
6294 + const switchedSelection = { instanceId: alternateInstanceId, model: "gpt-5.5" };
6295 + yield* orchestrator.dispatch({
6296 + type: "thread.create",
6297 + commandId: CommandId.make(`${threadId}:create`),
6298 + threadId,
6299 + createdBy: "user",
6300 + creationSource: "web",
6301 + projectId: ProjectId.make(`${threadId}:project`),
6302 + title: "Limit switch",
6303 + modelSelection,
6304 + runtimeMode: "full-access",
6305 + interactionMode: "default",
6306 + branch: null,
6307 + worktreePath: process.cwd(),
6308 + });
6309 + const send = (
6310 + name: string,
6311 + selection: typeof modelSelection | typeof switchedSelection,
6312 + type: "start_immediately" | "queue_after_active",
6313 + ) =>
6314 + orchestrator.dispatch({
6315 + type: "message.dispatch",
6316 + commandId: CommandId.make(`${threadId}:${name}`),
6317 + threadId,
6318 + messageId: MessageId.make(`${threadId}:${name}`),
6319 + text: name,
6320 + attachments: [],
6321 + createdBy: "user",
6322 + creationSource: "web",
6323 + modelSelection: selection,
6324 + dispatchMode: { type },
6325 + });
6326 + yield* send("limited", modelSelection, "start_immediately");
6327 + yield* send("queued", modelSelection, "queue_after_active");
6328 + const [limited, queued] = (yield* orchestrator.getThreadProjection(threadId)).runs;
6329 + assert.isDefined(limited);
6330 + assert.isDefined(queued);
6331 + const now = yield* DateTime.now;
6332 + yield* eventSink.write({
6333 + events: [
6334 + {
6335 + id: EventId.make(`${threadId}:error`),
6336 + type: "turn-item.updated",
6337 + threadId,
6338 + runId: limited.id,
6339 + nodeId: limited.rootNodeId!,
6340 + providerInstanceId: limited.providerInstanceId,
6341 + occurredAt: now,
6342 + payload: {
6343 + id: TurnItemId.make(`${threadId}:error`),
6344 + type: "error",
6345 + threadId,
6346 + runId: limited.id,
6347 + nodeId: limited.rootNodeId!,
6348 + providerThreadId: limited.providerThreadId,
6349 + providerTurnId: null,
6350 + nativeItemRef: null,
6351 + parentItemId: null,
6352 + ordinal: 2,
6353 + status: "failed",
6354 + title: "Usage limit",
6355 + startedAt: now,
6356 + completedAt: now,
6357 + updatedAt: now,
6358 + failure: {
6359 + class: "usage_limit",
6360 + message: "Usage limit reached",
6361 + code: "usage_limit",
6362 + retryable: null,
6363 + resetAt: DateTime.formatIso(DateTime.add(now, { hours: 1 })),
6364 + },
6365 + },
6366 + },
6367 + {
6368 + id: EventId.make(`${threadId}:failed`),
6369 + type: "run.updated",
6370 + threadId,
6371 + runId: limited.id,
6372 + providerInstanceId: limited.providerInstanceId,
6373 + occurredAt: now,
6374 + payload: { ...limited, status: "failed", completedAt: now },
6375 + },
6376 + ],
6377 + });
6378 +
6379 + // A message on another provider starts ahead of the held queue and
6380 + // completes there.
6381 + yield* send("switch", switchedSelection, "start_immediately");
6382 + const switched = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
6383 + (run) => run.userMessageId === MessageId.make(`${threadId}:switch`),
6384 + );
6385 + assert.equal(switched?.status, "starting");
6386 + yield* eventSink.write({
6387 + events: [
6388 + {
6389 + id: EventId.make(`${threadId}:switch:completed`),
6390 + type: "run.updated",
6391 + threadId,
6392 + runId: switched!.id,
6393 + providerInstanceId: switched!.providerInstanceId,
6394 + occurredAt: now,
6395 + payload: { ...switched!, status: "completed", completedAt: now },
6396 + },
6397 + ],
6398 + });
6399 +
6400 + yield* orchestrator.resumeQueuedRuns;
6401 + const held = yield* orchestrator.getThreadProjection(threadId);
6402 + assert.equal(held.runs.find((run) => run.id === queued.id)?.status, "queued");
6403 +
6404 + if (release === "the reset passes") {
6405 + yield* TestClock.adjust("61 minutes");
6406 + yield* orchestrator.resumeQueuedRuns;
6407 + } else {
6408 + yield* orchestrator.dispatch({
6409 + type: "queue.resume",
6410 + commandId: CommandId.make(`${threadId}:resume`),
6411 + threadId,
6412 + });
6413 + }
6414 + const reset = yield* orchestrator.getThreadProjection(threadId);
6415 + const resumed = reset.runs.find((run) => run.id === queued.id);
6416 + assert.equal(resumed?.status, "starting");
6417 + assert.deepEqual(resumed?.modelSelection, modelSelection);
6418 + }),
6419 + );
6420 +
6421 it.effect.each([
6422 ["server", modelSelection],
6423 ["provider", modelSelection],