The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 14:45 UTC
fix(server): turns stuck in provider retries for 10 minutes fail instead of running forever (#29208)
Co-authored-by: maria-rcks <maria@kuuro.net> Co-authored-by: Julius Marminge <julius0216@outlook.com>
Jex committed
Nov 14, 2026 at 23:10 UTC
66a40c7ca8223b2126a2a4baf94e7374169c0703
5 files changed
+269
-24
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
+168
@@ -11409,7 +11409,175 @@ describe("ClaudeAdapterV2 background wake turns", () => {
11409
}),
11410
);
11411
yield* harness.awaitTerminal(1);
11412
+ // A recovered retry must not fire its budget later.
11413
+ yield* TestClock.adjust("10 minutes");
11414
+ assert.lengthOf(harness.terminalEvents(), 1);
11415
+ assert.equal(harness.terminalEvents()[0]?.status, "completed");
11416
+ }).pipe(
11417
+ Effect.provide(
11418
+ Layer.mergeAll(
11419
+ IdAllocator.layer,
11420
+ McpProviderSessions.layer,
11421
+ NodeServices.layer,
11422
+ NodeCrypto.layer,
11423
+ ),
11424
+ ),
11425
+ ),
11426
+ );
11427
+
11428
+ const offerApiRetry = (harness: Effect.Success<typeof makeWakeHarness>, uuid: string) =>
11429
+ Effect.gen(function* () {
11430
+ yield* Queue.offer(
11431
+ harness.sdkMessages,
11432
+ claudeSdkFrame({
11433
+ type: "system",
11434
+ subtype: "api_retry",
11435
+ attempt: 1,
11436
+ max_retries: 10,
11437
+ retry_delay_ms: 600,
11438
+ error_status: 503,
11439
+ error: "server_error",
11440
+ uuid,
11441
+ session_id: WAKE_NATIVE_SESSION,
11442
+ }),
11443
+ );
11444
+ yield* harness.until(() =>
11445
+ harness.events.some(
11446
+ (event) =>
11447
+ event.type === "turn_item.updated" &&
11448
+ event.turnItem.type === "error" &&
11449
+ event.turnItem.status === "running",
11450
+ ),
11451
+ );
11452
+ });
11453
+
11454
+ it.effect.each(["closed", "interrupt-hangs", "unconfirmed", "close-hangs", "synthetic"] as const)(
11455
+ "fails a turn whose API retries outlast the retry budget: %s",
11456
+ (response) =>
11457
+ Effect.gen(function* () {
11458
+ const interruptSent = yield* Deferred.make<void>();
11459
+ const closeRequested = yield* Deferred.make<void>();
11460
+ const releaseClose = yield* Deferred.make<void>();
11461
+ const harness = yield* makeWakeHarnessWithOptions({
11462
+ interrupt: Deferred.succeed(interruptSent, undefined).pipe(
11463
+ Effect.andThen(response === "interrupt-hangs" ? Effect.never : Effect.void),
11464
+ ),
11465
+ close: (sdkMessages) =>
11466
+ Deferred.succeed(closeRequested, undefined).pipe(
11467
+ Effect.andThen(
11468
+ response === "close-hangs"
11469
+ ? Deferred.await(releaseClose).pipe(Effect.andThen(Queue.shutdown(sdkMessages)))
11470
+ : response === "unconfirmed"
11471
+ ? Effect.void
11472
+ : Queue.shutdown(sdkMessages),
11473
+ ),
11474
+ ),
11475
+ });
11476
+ yield* Effect.addFinalizer(() => Deferred.succeed(releaseClose, undefined));
11477
+ yield* harness.runtime.startTurn(
11478
+ makeClaudeTestTurnInput({
11479
+ threadId: harness.threadId,
11480
+ providerThread: harness.providerThread,
11481
+ now: yield* DateTime.now,
11482
+ attemptId: RunAttemptId.make("attempt-claude-retry-budget"),
11483
+ text: "Summarize the repo.",
11484
+ attachments: [],
11485
+ }),
11486
+ );
11487
+ yield* offerApiRetry(harness, "00000000-0000-4000-8000-000000000211");
11488
+ if (response === "synthetic") {
11489
+ // The CLI reports the API error as a <synthetic> assistant frame; that is not recovery.
11490
+ yield* Queue.offer(
11491
+ harness.sdkMessages,
11492
+ claudeSdkFrame({
11493
+ type: "assistant",
11494
+ message: {
11495
+ model: "<synthetic>",
11496
+ id: "retry-notice",
11497
+ type: "message",
11498
+ role: "assistant",
11499
+ content: [{ type: "text", text: "API Error: 503" }],
11500
+ stop_reason: "end_turn",
11501
+ stop_sequence: null,
11502
+ usage: {
11503
+ input_tokens: 0,
11504
+ output_tokens: 0,
11505
+ cache_creation_input_tokens: 0,
11506
+ cache_read_input_tokens: 0,
11507
+ },
11508
+ },
11509
+ parent_tool_use_id: null,
11510
+ uuid: "retry-notice",
11511
+ session_id: WAKE_NATIVE_SESSION,
11512
+ }),
11513
+ );
11514
+ yield* harness.until(() =>
11515
+ harness.events.some(
11516
+ (event) =>
11517
+ event.type === "turn_item.updated" && event.turnItem.type === "assistant_message",
11518
+ ),
11519
+ );
11520
+ }
11521
+ yield* TestClock.adjust("9 minutes");
11522
+ assert.lengthOf(harness.terminalEvents(), 0);
11523
+ yield* TestClock.adjust("1 minute");
11524
+ yield* Deferred.await(interruptSent);
11525
+ if (response !== "closed" && response !== "synthetic") {
11526
+ yield* TestClock.adjust("20 seconds");
11527
+ }
11528
+ const terminal = yield* Queue.take(harness.terminalReceipts);
11529
+ assert.equal(terminal.status, "failed");
11530
+ if (terminal.status !== "failed") return;
11531
+ assert.equal(terminal.failure.code, "api_error_503");
11532
+ assert.equal(terminal.retry?.attempt, 1);
11533
+ assert.lengthOf(harness.terminalEvents(), 1);
11534
+ assert.isTrue(yield* Deferred.isDone(closeRequested), "the query must be closed");
11535
+ }).pipe(
11536
+ Effect.scoped,
11537
+ Effect.provide(
11538
+ Layer.mergeAll(
11539
+ IdAllocator.layer,
11540
+ McpProviderSessions.layer,
11541
+ NodeServices.layer,
11542
+ NodeCrypto.layer,
11543
+ ),
11544
+ ),
11545
+ ),
11546
+ );
11547
+
11548
+ it.effect("keeps a user Stop interrupted when the retry budget passes before it settles", () =>
11549
+ Effect.gen(function* () {
11550
+ const harness = yield* makeWakeHarness;
11551
+ yield* harness.runtime.startTurn(
11552
+ makeClaudeTestTurnInput({
11553
+ threadId: harness.threadId,
11554
+ providerThread: harness.providerThread,
11555
+ now: yield* DateTime.now,
11556
+ attemptId: RunAttemptId.make("attempt-claude-retry-budget-user-stop"),
11557
+ text: "Summarize the repo.",
11558
+ attachments: [],
11559
+ }),
11560
+ );
11561
+ yield* offerApiRetry(harness, "00000000-0000-4000-8000-000000000212");
11562
+ const active = harness.events.find(
11563
+ (event) =>
11564
+ event.type === "provider_turn.updated" && event.providerTurn.status === "running",
11565
+ );
11566
+ if (active?.type !== "provider_turn.updated")
11567
+ return yield* Effect.die("Missing running native turn");
11568
+ // The CLI has not confirmed the Stop when the budget passes.
11569
+ yield* harness.runtime
11570
+ .interruptTurn({
11571
+ providerThread: harness.providerThread,
11572
+ providerTurnId: active.providerTurn.id,
11573
+ })
11574
+ .pipe(Effect.ignore, Effect.forkScoped);
11575
+ yield* TestClock.adjust("10 minutes");
11576
+ yield* Queue.shutdown(harness.sdkMessages);
11577
+ assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "interrupted");
11578
+ assert.lengthOf(harness.terminalEvents(), 1);
11579
}).pipe(
11580
+ Effect.scoped,
11581
Effect.provide(
11582
Layer.mergeAll(
11583
IdAllocator.layer,
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+64
-3
@@ -227,6 +227,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
227
import {
228
makeProviderFailure,
229
makeProviderRetryTurnItem,
230
+ PROVIDER_RETRY_BUDGET_MS,
231
} from "@t3tools/provider-core/server/failure";
232
import { turnScopedSelectionTransition } from "@t3tools/provider-core/server/selectionTransition";
233
import { providerMessageTextWithAttachmentPaths } from "@t3tools/provider-core/server/attachmentPrompt";
@@ -4813,6 +4814,7 @@ function formatClaudeUsageLimitWait(waitMs: number): string {
4814
}
4815
4816
interface ActiveClaudeTurnContext {
4817
+ retryBudgetFailure?: OrchestrationV2ProviderFailure;
4818
readonly queryPolicy: ClaudeRuntimeQueryPolicy;
4819
readonly input: ProviderAdapter.ProviderAdapterV2TurnInput;
4820
readonly autoCompactWindow: string;
@@ -6213,6 +6215,51 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
6215
}).pipe(Effect.withSpan("ClaudeAdapterV2.completeProviderRetry"));
6216
});
6217
6218
+ // A hung request can take many minutes per attempt, so the budget spans the
6219
+ // whole retry window rather than counting attempts.
6220
+ const failExpiredProviderRetry = Effect.fn("ClaudeAdapterV2.failExpiredProviderRetry")(
6221
+ function* (context: ActiveClaudeTurnContext, startedAt: DateTime.Utc) {
6222
+ const isCurrentRetry = Effect.gen(function* () {
6223
+ const retry = (yield* Ref.get(providerRetries)).get(context.providerTurnId);
6224
+ return retry?.startedAt === startedAt && (yield* Ref.get(activeTurn)) === context
6225
+ ? retry
6226
+ : undefined;
6227
+ });
6228
+ const retry = yield* isCurrentRetry;
6229
+ // A user Stop already in flight keeps its interrupted outcome.
6230
+ if (
6231
+ retry === undefined ||
6232
+ (yield* Ref.get(interruptedTurns)).has(context.providerTurnId)
6233
+ )
6234
+ return;
6235
+ context.retryBudgetFailure = { ...retry.failure, retryable: false };
6236
+ const liveQuery = yield* Ref.get(queryContext);
6237
+ yield* interruptTurn({
6238
+ providerThread: context.input.providerThread,
6239
+ providerTurnId: context.providerTurnId,
6240
+ }).pipe(
6241
+ Effect.timeoutOption(CLAUDE_STOP_CONFIRM_TIMEOUT_MS),
6242
+ Effect.catch((cause) =>
6243
+ Effect.logWarning("orchestration-v2.claude-retry-budget-stop-failed", { cause }),
6244
+ ),
6245
+ );
6246
+ if (liveQuery === null || (yield* isCurrentRetry) === undefined) return;
6247
+ // The CLI did not confirm the stop; settle the turn without it.
6248
+ const ownedLiveQuery = yield* Ref.modify(queryContext, (current) =>
6249
+ current?.query === liveQuery.query ? [true, null] : [false, current],
6250
+ );
6251
+ if (!ownedLiveQuery) return;
6252
+ liveQuery.stopping = true;
6253
+ yield* liveQuery.query.close.pipe(
6254
+ Effect.timeoutOption(CLAUDE_STOP_CONFIRM_TIMEOUT_MS),
6255
+ Effect.ignore,
6256
+ );
6257
+ yield* finalizeActiveTurnAfterQueryExit(liveQuery);
6258
+ yield* endSubagentCallsOfClosedQuery(liveQuery.nativeThreadId, "interrupted");
6259
+ yield* Deferred.succeed(liveQuery.closed, undefined);
6260
+ },
6261
+ );
6262
+
6263
const rememberProviderThread =
6264
/* Provider events include streaming deltas; avoid a span for each update. */
6265
Effect.fnUntraced(function* (providerThread: OrchestrationV2ProviderThread) {
@@ -8879,7 +8926,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
8926
}
8927
});
8928
8882
- const finalizeActiveTurn = Effect.fnUntraced(function* (input: {
8929
+ const finalizeActiveTurn = Effect.fnUntraced(function* (requested: {
8930
readonly context: ActiveClaudeTurnContext;
8931
readonly status: Extract<
8932
OrchestrationV2ProviderTurn["status"],
@@ -8890,6 +8937,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
8937
readonly threadDisposition?: "reusable" | "broken";
8938
readonly result?: SDKResultMessage;
8939
}) {
8940
+ // A turn stopped by the retry budget settles as failed with the retry's error.
8941
+ const retryBudgetFailure = requested.context.retryBudgetFailure;
8942
+ const input =
8943
+ requested.status === "interrupted" && retryBudgetFailure !== undefined
8944
+ ? { ...requested, status: "failed" as const, failure: retryBudgetFailure }
8945
+ : requested;
8946
input.context.steersClosed = true;
8947
yield* reasoningDeltas.flushTurn(input.context.nativeTurnId);
8948
if (input.status === "completed") {
@@ -10747,12 +10800,20 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
10800
updatedAt,
10801
}),
10802
});
10803
+ if (previous === undefined) {
10804
+ yield* failExpiredProviderRetry(context, state.startedAt).pipe(
10805
+ Effect.delay(PROVIDER_RETRY_BUDGET_MS),
10806
+ Effect.forkIn(sessionScope),
10807
+ );
10808
+ }
10809
return;
10810
}
10811
10812
if (message.type === "assistant" && input.replayed !== true) {
10754
- const now = yield* DateTime.now;
10755
- yield* completeProviderRetry(context, now);
10813
+ if (message.message.model !== "<synthetic>") {
10814
+ const now = yield* DateTime.now;
10815
+ yield* completeProviderRetry(context, now);
10816
+ }
10817
// CLI-generated <synthetic> frames report zero usage, not context usage.
10818
if (
10819
message.parent_tool_use_id === null &&
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts
+21
-18
@@ -4596,27 +4596,30 @@ describe("CodexAdapterV2 post-settle continuation", () => {
4596
const scenario = "codex-provider-retry-stall";
4597
const nativeThreadId = `native-${scenario}-thread`;
4598
const nativeTurnId = `native-${scenario}-turn`;
4599
+ const retryNotice = (attempt: number) => ({
4600
+ type: "emit_inbound" as const,
4601
+ label: `error/retry-${attempt}`,
4602
+ frame: {
4603
+ method: "error",
4604
+ params: {
4605
+ threadId: nativeThreadId,
4606
+ turnId: nativeTurnId,
4607
+ willRetry: true,
4608
+ error: {
4609
+ message: `Reconnecting... ${attempt}/5`,
4610
+ additionalDetails: "exceeded retry limit, last status: 429 Too Many Requests",
4611
+ codexErrorInfo: { responseStreamDisconnected: { httpStatusCode: 429 } },
4612
+ },
4613
+ },
4614
+ },
4615
+ });
4616
const transcript = makeCodexReplayTranscript({
4617
scenario,
4618
entries: [
4619
...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt: "Continue." }),
4603
- {
4604
- type: "emit_inbound",
4605
- label: "error/retry",
4606
- frame: {
4607
- method: "error",
4608
- params: {
4609
- threadId: nativeThreadId,
4610
- turnId: nativeTurnId,
4611
- willRetry: true,
4612
- error: {
4613
- message: "Reconnecting... 1/5",
4614
- additionalDetails: "exceeded retry limit, last status: 429 Too Many Requests",
4615
- codexErrorInfo: { responseStreamDisconnected: { httpStatusCode: 429 } },
4616
- },
4617
- },
4618
- },
4619
- },
4620
+ retryNotice(1),
4621
+ // A later notice in the same retry window must not push the deadline back.
4622
+ { ...retryNotice(2), afterMs: 5 * 60 * 1000 },
4623
{
4624
type: "expect_outbound",
4625
label: "turn/interrupt",
@@ -4675,7 +4678,7 @@ describe("CodexAdapterV2 post-settle continuation", () => {
4678
terminal.failure.message,
4679
"exceeded retry limit, last status: 429 Too Many Requests",
4680
);
4678
- assert.equal(terminal.retry?.attempt, 1);
4681
+ assert.equal(terminal.retry?.attempt, 2);
4682
assert.equal(terminal.threadDisposition, "reusable");
4683
}).pipe(
4684
Effect.provide(
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
+9
-3
@@ -144,6 +144,7 @@ import {
144
makeProviderFailure,
145
makeProviderFailureTurnItem,
146
makeProviderRetryTurnItem,
147
+ PROVIDER_RETRY_BUDGET_MS,
148
} from "@t3tools/provider-core/server/failure";
149
import { turnScopedSelectionTransition } from "@t3tools/provider-core/server/selectionTransition";
150
import {
@@ -162,8 +163,6 @@ const CODEX_PROVIDER = ProviderDriverKind.make("codex");
163
const CODEX_CAPACITY_MAX_RETRIES = 5;
164
// How long Stop waits for Codex to end the turn before the user is offered Force stop.
165
const CODEX_STOP_CONFIRM_TIMEOUT_MS = 10_000;
165
-// Codex can announce a retry and never resume. Recovered retries resume model output within ~75s.
166
-const CODEX_PROVIDER_RETRY_STALL_TIMEOUT = "10 minutes";
166
export const CODEX_DRIVER_KIND = CODEX_PROVIDER;
167
export const CODEX_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(CODEX_DRIVER_KIND);
168
@@ -5173,7 +5172,14 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
5172
return updated;
5173
});
5174
yield* failStalledProviderRetry(context, state, payload.error.codexErrorInfo).pipe(
5176
- Effect.delay(CODEX_PROVIDER_RETRY_STALL_TIMEOUT),
5175
+ // Every notice aims at the same deadline, so repeated notices cannot extend it.
5176
+ Effect.delay(
5177
+ Math.max(
5178
+ 0,
5179
+ PROVIDER_RETRY_BUDGET_MS -
5180
+ (DateTime.toEpochMillis(updatedAt) - DateTime.toEpochMillis(state.startedAt)),
5181
+ ),
5182
+ ),
5183
Effect.forkIn(scope, { startImmediately: true }),
5184
);
5185
yield* emitProviderEvent({
packages/provider-core/src/server/failure.ts
+7
@@ -333,6 +333,13 @@ export function makeProviderFailureTurnItem(input: {
333
};
334
}
335
336
+/**
337
+ * Longest a turn may stay in provider retries before the adapter stops it as
338
+ * failed. Codex waits for the network without limit and a hung Claude request
339
+ * can take many minutes per attempt, so counting attempts alone never ends them.
340
+ */
341
+export const PROVIDER_RETRY_BUDGET_MS = 10 * 60 * 1000;
342
+
343
export function makeProviderRetryTurnItem(input: {
344
readonly driver: ProviderDriverKind;
345
readonly threadId: ThreadId;