@devries-cameron20 / t3code-1 / commits / 66a40c7ca8
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): turns stuck in provider retries for 10 minutes fail instead of running forever (#29208)

Co-authored-by: maria-rcks <maria@kuuro.net> Co-authored-by: Julius Marminge <julius0216@outlook.com>

Jex committed Nov 14, 2026 at 23:10 UTC 66a40c7ca8223b2126a2a4baf94e7374169c0703
5 files changed +269 -24
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
+168
@@ -11409,7 +11409,175 @@ describe("ClaudeAdapterV2 background wake turns", () => {
11409 }),
11410 );
11411 yield* harness.awaitTerminal(1);
11412 + // A recovered retry must not fire its budget later.
11413 + yield* TestClock.adjust("10 minutes");
11414 + assert.lengthOf(harness.terminalEvents(), 1);
11415 + assert.equal(harness.terminalEvents()[0]?.status, "completed");
11416 + }).pipe(
11417 + Effect.provide(
11418 + Layer.mergeAll(
11419 + IdAllocator.layer,
11420 + McpProviderSessions.layer,
11421 + NodeServices.layer,
11422 + NodeCrypto.layer,
11423 + ),
11424 + ),
11425 + ),
11426 + );
11427 +
11428 + const offerApiRetry = (harness: Effect.Success<typeof makeWakeHarness>, uuid: string) =>
11429 + Effect.gen(function* () {
11430 + yield* Queue.offer(
11431 + harness.sdkMessages,
11432 + claudeSdkFrame({
11433 + type: "system",
11434 + subtype: "api_retry",
11435 + attempt: 1,
11436 + max_retries: 10,
11437 + retry_delay_ms: 600,
11438 + error_status: 503,
11439 + error: "server_error",
11440 + uuid,
11441 + session_id: WAKE_NATIVE_SESSION,
11442 + }),
11443 + );
11444 + yield* harness.until(() =>
11445 + harness.events.some(
11446 + (event) =>
11447 + event.type === "turn_item.updated" &&
11448 + event.turnItem.type === "error" &&
11449 + event.turnItem.status === "running",
11450 + ),
11451 + );
11452 + });
11453 +
11454 + it.effect.each(["closed", "interrupt-hangs", "unconfirmed", "close-hangs", "synthetic"] as const)(
11455 + "fails a turn whose API retries outlast the retry budget: %s",
11456 + (response) =>
11457 + Effect.gen(function* () {
11458 + const interruptSent = yield* Deferred.make<void>();
11459 + const closeRequested = yield* Deferred.make<void>();
11460 + const releaseClose = yield* Deferred.make<void>();
11461 + const harness = yield* makeWakeHarnessWithOptions({
11462 + interrupt: Deferred.succeed(interruptSent, undefined).pipe(
11463 + Effect.andThen(response === "interrupt-hangs" ? Effect.never : Effect.void),
11464 + ),
11465 + close: (sdkMessages) =>
11466 + Deferred.succeed(closeRequested, undefined).pipe(
11467 + Effect.andThen(
11468 + response === "close-hangs"
11469 + ? Deferred.await(releaseClose).pipe(Effect.andThen(Queue.shutdown(sdkMessages)))
11470 + : response === "unconfirmed"
11471 + ? Effect.void
11472 + : Queue.shutdown(sdkMessages),
11473 + ),
11474 + ),
11475 + });
11476 + yield* Effect.addFinalizer(() => Deferred.succeed(releaseClose, undefined));
11477 + yield* harness.runtime.startTurn(
11478 + makeClaudeTestTurnInput({
11479 + threadId: harness.threadId,
11480 + providerThread: harness.providerThread,
11481 + now: yield* DateTime.now,
11482 + attemptId: RunAttemptId.make("attempt-claude-retry-budget"),
11483 + text: "Summarize the repo.",
11484 + attachments: [],
11485 + }),
11486 + );
11487 + yield* offerApiRetry(harness, "00000000-0000-4000-8000-000000000211");
11488 + if (response === "synthetic") {
11489 + // The CLI reports the API error as a <synthetic> assistant frame; that is not recovery.
11490 + yield* Queue.offer(
11491 + harness.sdkMessages,
11492 + claudeSdkFrame({
11493 + type: "assistant",
11494 + message: {
11495 + model: "<synthetic>",
11496 + id: "retry-notice",
11497 + type: "message",
11498 + role: "assistant",
11499 + content: [{ type: "text", text: "API Error: 503" }],
11500 + stop_reason: "end_turn",
11501 + stop_sequence: null,
11502 + usage: {
11503 + input_tokens: 0,
11504 + output_tokens: 0,
11505 + cache_creation_input_tokens: 0,
11506 + cache_read_input_tokens: 0,
11507 + },
11508 + },
11509 + parent_tool_use_id: null,
11510 + uuid: "retry-notice",
11511 + session_id: WAKE_NATIVE_SESSION,
11512 + }),
11513 + );
11514 + yield* harness.until(() =>
11515 + harness.events.some(
11516 + (event) =>
11517 + event.type === "turn_item.updated" && event.turnItem.type === "assistant_message",
11518 + ),
11519 + );
11520 + }
11521 + yield* TestClock.adjust("9 minutes");
11522 + assert.lengthOf(harness.terminalEvents(), 0);
11523 + yield* TestClock.adjust("1 minute");
11524 + yield* Deferred.await(interruptSent);
11525 + if (response !== "closed" && response !== "synthetic") {
11526 + yield* TestClock.adjust("20 seconds");
11527 + }
11528 + const terminal = yield* Queue.take(harness.terminalReceipts);
11529 + assert.equal(terminal.status, "failed");
11530 + if (terminal.status !== "failed") return;
11531 + assert.equal(terminal.failure.code, "api_error_503");
11532 + assert.equal(terminal.retry?.attempt, 1);
11533 + assert.lengthOf(harness.terminalEvents(), 1);
11534 + assert.isTrue(yield* Deferred.isDone(closeRequested), "the query must be closed");
11535 + }).pipe(
11536 + Effect.scoped,
11537 + Effect.provide(
11538 + Layer.mergeAll(
11539 + IdAllocator.layer,
11540 + McpProviderSessions.layer,
11541 + NodeServices.layer,
11542 + NodeCrypto.layer,
11543 + ),
11544 + ),
11545 + ),
11546 + );
11547 +
11548 + it.effect("keeps a user Stop interrupted when the retry budget passes before it settles", () =>
11549 + Effect.gen(function* () {
11550 + const harness = yield* makeWakeHarness;
11551 + yield* harness.runtime.startTurn(
11552 + makeClaudeTestTurnInput({
11553 + threadId: harness.threadId,
11554 + providerThread: harness.providerThread,
11555 + now: yield* DateTime.now,
11556 + attemptId: RunAttemptId.make("attempt-claude-retry-budget-user-stop"),
11557 + text: "Summarize the repo.",
11558 + attachments: [],
11559 + }),
11560 + );
11561 + yield* offerApiRetry(harness, "00000000-0000-4000-8000-000000000212");
11562 + const active = harness.events.find(
11563 + (event) =>
11564 + event.type === "provider_turn.updated" && event.providerTurn.status === "running",
11565 + );
11566 + if (active?.type !== "provider_turn.updated")
11567 + return yield* Effect.die("Missing running native turn");
11568 + // The CLI has not confirmed the Stop when the budget passes.
11569 + yield* harness.runtime
11570 + .interruptTurn({
11571 + providerThread: harness.providerThread,
11572 + providerTurnId: active.providerTurn.id,
11573 + })
11574 + .pipe(Effect.ignore, Effect.forkScoped);
11575 + yield* TestClock.adjust("10 minutes");
11576 + yield* Queue.shutdown(harness.sdkMessages);
11577 + assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "interrupted");
11578 + assert.lengthOf(harness.terminalEvents(), 1);
11579 }).pipe(
11580 + Effect.scoped,
11581 Effect.provide(
11582 Layer.mergeAll(
11583 IdAllocator.layer,
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+64 -3
@@ -227,6 +227,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
227 import {
228 makeProviderFailure,
229 makeProviderRetryTurnItem,
230 + PROVIDER_RETRY_BUDGET_MS,
231 } from "@t3tools/provider-core/server/failure";
232 import { turnScopedSelectionTransition } from "@t3tools/provider-core/server/selectionTransition";
233 import { providerMessageTextWithAttachmentPaths } from "@t3tools/provider-core/server/attachmentPrompt";
@@ -4813,6 +4814,7 @@ function formatClaudeUsageLimitWait(waitMs: number): string {
4814 }
4815
4816 interface ActiveClaudeTurnContext {
4817 + retryBudgetFailure?: OrchestrationV2ProviderFailure;
4818 readonly queryPolicy: ClaudeRuntimeQueryPolicy;
4819 readonly input: ProviderAdapter.ProviderAdapterV2TurnInput;
4820 readonly autoCompactWindow: string;
@@ -6213,6 +6215,51 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
6215 }).pipe(Effect.withSpan("ClaudeAdapterV2.completeProviderRetry"));
6216 });
6217
6218 + // A hung request can take many minutes per attempt, so the budget spans the
6219 + // whole retry window rather than counting attempts.
6220 + const failExpiredProviderRetry = Effect.fn("ClaudeAdapterV2.failExpiredProviderRetry")(
6221 + function* (context: ActiveClaudeTurnContext, startedAt: DateTime.Utc) {
6222 + const isCurrentRetry = Effect.gen(function* () {
6223 + const retry = (yield* Ref.get(providerRetries)).get(context.providerTurnId);
6224 + return retry?.startedAt === startedAt && (yield* Ref.get(activeTurn)) === context
6225 + ? retry
6226 + : undefined;
6227 + });
6228 + const retry = yield* isCurrentRetry;
6229 + // A user Stop already in flight keeps its interrupted outcome.
6230 + if (
6231 + retry === undefined ||
6232 + (yield* Ref.get(interruptedTurns)).has(context.providerTurnId)
6233 + )
6234 + return;
6235 + context.retryBudgetFailure = { ...retry.failure, retryable: false };
6236 + const liveQuery = yield* Ref.get(queryContext);
6237 + yield* interruptTurn({
6238 + providerThread: context.input.providerThread,
6239 + providerTurnId: context.providerTurnId,
6240 + }).pipe(
6241 + Effect.timeoutOption(CLAUDE_STOP_CONFIRM_TIMEOUT_MS),
6242 + Effect.catch((cause) =>
6243 + Effect.logWarning("orchestration-v2.claude-retry-budget-stop-failed", { cause }),
6244 + ),
6245 + );
6246 + if (liveQuery === null || (yield* isCurrentRetry) === undefined) return;
6247 + // The CLI did not confirm the stop; settle the turn without it.
6248 + const ownedLiveQuery = yield* Ref.modify(queryContext, (current) =>
6249 + current?.query === liveQuery.query ? [true, null] : [false, current],
6250 + );
6251 + if (!ownedLiveQuery) return;
6252 + liveQuery.stopping = true;
6253 + yield* liveQuery.query.close.pipe(
6254 + Effect.timeoutOption(CLAUDE_STOP_CONFIRM_TIMEOUT_MS),
6255 + Effect.ignore,
6256 + );
6257 + yield* finalizeActiveTurnAfterQueryExit(liveQuery);
6258 + yield* endSubagentCallsOfClosedQuery(liveQuery.nativeThreadId, "interrupted");
6259 + yield* Deferred.succeed(liveQuery.closed, undefined);
6260 + },
6261 + );
6262 +
6263 const rememberProviderThread =
6264 /* Provider events include streaming deltas; avoid a span for each update. */
6265 Effect.fnUntraced(function* (providerThread: OrchestrationV2ProviderThread) {
@@ -8879,7 +8926,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
8926 }
8927 });
8928
8882 - const finalizeActiveTurn = Effect.fnUntraced(function* (input: {
8929 + const finalizeActiveTurn = Effect.fnUntraced(function* (requested: {
8930 readonly context: ActiveClaudeTurnContext;
8931 readonly status: Extract<
8932 OrchestrationV2ProviderTurn["status"],
@@ -8890,6 +8937,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
8937 readonly threadDisposition?: "reusable" | "broken";
8938 readonly result?: SDKResultMessage;
8939 }) {
8940 + // A turn stopped by the retry budget settles as failed with the retry's error.
8941 + const retryBudgetFailure = requested.context.retryBudgetFailure;
8942 + const input =
8943 + requested.status === "interrupted" && retryBudgetFailure !== undefined
8944 + ? { ...requested, status: "failed" as const, failure: retryBudgetFailure }
8945 + : requested;
8946 input.context.steersClosed = true;
8947 yield* reasoningDeltas.flushTurn(input.context.nativeTurnId);
8948 if (input.status === "completed") {
@@ -10747,12 +10800,20 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
10800 updatedAt,
10801 }),
10802 });
10803 + if (previous === undefined) {
10804 + yield* failExpiredProviderRetry(context, state.startedAt).pipe(
10805 + Effect.delay(PROVIDER_RETRY_BUDGET_MS),
10806 + Effect.forkIn(sessionScope),
10807 + );
10808 + }
10809 return;
10810 }
10811
10812 if (message.type === "assistant" && input.replayed !== true) {
10754 - const now = yield* DateTime.now;
10755 - yield* completeProviderRetry(context, now);
10813 + if (message.message.model !== "<synthetic>") {
10814 + const now = yield* DateTime.now;
10815 + yield* completeProviderRetry(context, now);
10816 + }
10817 // CLI-generated <synthetic> frames report zero usage, not context usage.
10818 if (
10819 message.parent_tool_use_id === null &&
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts
+21 -18
@@ -4596,27 +4596,30 @@ describe("CodexAdapterV2 post-settle continuation", () => {
4596 const scenario = "codex-provider-retry-stall";
4597 const nativeThreadId = `native-${scenario}-thread`;
4598 const nativeTurnId = `native-${scenario}-turn`;
4599 + const retryNotice = (attempt: number) => ({
4600 + type: "emit_inbound" as const,
4601 + label: `error/retry-${attempt}`,
4602 + frame: {
4603 + method: "error",
4604 + params: {
4605 + threadId: nativeThreadId,
4606 + turnId: nativeTurnId,
4607 + willRetry: true,
4608 + error: {
4609 + message: `Reconnecting... ${attempt}/5`,
4610 + additionalDetails: "exceeded retry limit, last status: 429 Too Many Requests",
4611 + codexErrorInfo: { responseStreamDisconnected: { httpStatusCode: 429 } },
4612 + },
4613 + },
4614 + },
4615 + });
4616 const transcript = makeCodexReplayTranscript({
4617 scenario,
4618 entries: [
4619 ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt: "Continue." }),
4603 - {
4604 - type: "emit_inbound",
4605 - label: "error/retry",
4606 - frame: {
4607 - method: "error",
4608 - params: {
4609 - threadId: nativeThreadId,
4610 - turnId: nativeTurnId,
4611 - willRetry: true,
4612 - error: {
4613 - message: "Reconnecting... 1/5",
4614 - additionalDetails: "exceeded retry limit, last status: 429 Too Many Requests",
4615 - codexErrorInfo: { responseStreamDisconnected: { httpStatusCode: 429 } },
4616 - },
4617 - },
4618 - },
4619 - },
4620 + retryNotice(1),
4621 + // A later notice in the same retry window must not push the deadline back.
4622 + { ...retryNotice(2), afterMs: 5 * 60 * 1000 },
4623 {
4624 type: "expect_outbound",
4625 label: "turn/interrupt",
@@ -4675,7 +4678,7 @@ describe("CodexAdapterV2 post-settle continuation", () => {
4678 terminal.failure.message,
4679 "exceeded retry limit, last status: 429 Too Many Requests",
4680 );
4678 - assert.equal(terminal.retry?.attempt, 1);
4681 + assert.equal(terminal.retry?.attempt, 2);
4682 assert.equal(terminal.threadDisposition, "reusable");
4683 }).pipe(
4684 Effect.provide(
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
+9 -3
@@ -144,6 +144,7 @@ import {
144 makeProviderFailure,
145 makeProviderFailureTurnItem,
146 makeProviderRetryTurnItem,
147 + PROVIDER_RETRY_BUDGET_MS,
148 } from "@t3tools/provider-core/server/failure";
149 import { turnScopedSelectionTransition } from "@t3tools/provider-core/server/selectionTransition";
150 import {
@@ -162,8 +163,6 @@ const CODEX_PROVIDER = ProviderDriverKind.make("codex");
163 const CODEX_CAPACITY_MAX_RETRIES = 5;
164 // How long Stop waits for Codex to end the turn before the user is offered Force stop.
165 const CODEX_STOP_CONFIRM_TIMEOUT_MS = 10_000;
165 -// Codex can announce a retry and never resume. Recovered retries resume model output within ~75s.
166 -const CODEX_PROVIDER_RETRY_STALL_TIMEOUT = "10 minutes";
166 export const CODEX_DRIVER_KIND = CODEX_PROVIDER;
167 export const CODEX_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(CODEX_DRIVER_KIND);
168
@@ -5173,7 +5172,14 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
5172 return updated;
5173 });
5174 yield* failStalledProviderRetry(context, state, payload.error.codexErrorInfo).pipe(
5176 - Effect.delay(CODEX_PROVIDER_RETRY_STALL_TIMEOUT),
5175 + // Every notice aims at the same deadline, so repeated notices cannot extend it.
5176 + Effect.delay(
5177 + Math.max(
5178 + 0,
5179 + PROVIDER_RETRY_BUDGET_MS -
5180 + (DateTime.toEpochMillis(updatedAt) - DateTime.toEpochMillis(state.startedAt)),
5181 + ),
5182 + ),
5183 Effect.forkIn(scope, { startImmediately: true }),
5184 );
5185 yield* emitProviderEvent({
packages/provider-core/src/server/failure.ts
+7
@@ -333,6 +333,13 @@ export function makeProviderFailureTurnItem(input: {
333 };
334 }
335
336 +/**
337 + * Longest a turn may stay in provider retries before the adapter stops it as
338 + * failed. Codex waits for the network without limit and a hung Claude request
339 + * can take many minutes per attempt, so counting attempts alone never ends them.
340 + */
341 +export const PROVIDER_RETRY_BUDGET_MS = 10 * 60 * 1000;
342 +
343 export function makeProviderRetryTurnItem(input: {
344 readonly driver: ProviderDriverKind;
345 readonly threadId: ThreadId;