The best way to control Claude Code, Codex and more from any computer or phone
fix(server): fail Codex turns stuck in a provider retry (#33130)
Co-authored-by: Julius Marminge <julius0216@outlook.com>
maria committed
Nov 11, 2026 at 19:58 UTC
362ac8d6d5ced16089b98b30f4638e7577ca6581
2 files changed
+173
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts
+100
@@ -3826,6 +3826,106 @@ describe("CodexAdapterV2 post-settle continuation", () => {
3826
),
3827
);
3828
3829
+ it.effect("fails a stalled 429 provider retry as a usage limit and stops the Codex turn", () =>
3830
+ Effect.scoped(
3831
+ Effect.gen(function* () {
3832
+ const scenario = "codex-provider-retry-stall";
3833
+ const nativeThreadId = `native-${scenario}-thread`;
3834
+ const nativeTurnId = `native-${scenario}-turn`;
3835
+ const transcript = makeCodexReplayTranscript({
3836
+ scenario,
3837
+ entries: [
3838
+ ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt: "Continue." }),
3839
+ {
3840
+ type: "emit_inbound",
3841
+ label: "error/retry",
3842
+ frame: {
3843
+ method: "error",
3844
+ params: {
3845
+ threadId: nativeThreadId,
3846
+ turnId: nativeTurnId,
3847
+ willRetry: true,
3848
+ error: {
3849
+ message: "Reconnecting... 1/5",
3850
+ additionalDetails: "exceeded retry limit, last status: 429 Too Many Requests",
3851
+ codexErrorInfo: { responseStreamDisconnected: { httpStatusCode: 429 } },
3852
+ },
3853
+ },
3854
+ },
3855
+ },
3856
+ {
3857
+ type: "expect_outbound",
3858
+ label: "turn/interrupt",
3859
+ frame: {
3860
+ id: 4,
3861
+ method: "turn/interrupt",
3862
+ params: { threadId: nativeThreadId, turnId: nativeTurnId },
3863
+ },
3864
+ },
3865
+ { type: "emit_inbound", label: "turn/interrupt", frame: { id: 4, result: {} } },
3866
+ {
3867
+ type: "emit_inbound",
3868
+ label: "turn/completed",
3869
+ frame: {
3870
+ method: "turn/completed",
3871
+ params: {
3872
+ threadId: nativeThreadId,
3873
+ turn: makeCodexReplayTurn({ id: nativeTurnId, status: "interrupted" }),
3874
+ },
3875
+ },
3876
+ },
3877
+ ],
3878
+ });
3879
+ const retryShown = yield* Deferred.make<void>();
3880
+ const interruptSent = yield* Deferred.make<void>();
3881
+ const harness = yield* makeCodexReplayHarness(
3882
+ transcript,
3883
+ (event) =>
3884
+ event.type === "turn_item.updated" && event.turnItem.type === "error"
3885
+ ? Deferred.succeed(retryShown, undefined)
3886
+ : Effect.void,
3887
+ (method) =>
3888
+ method === "turn/interrupt" ? Deferred.succeed(interruptSent, undefined) : Effect.void,
3889
+ );
3890
+ yield* harness.runtime.startTurn(
3891
+ makeCodexTestTurnInput({
3892
+ threadId: harness.threadId,
3893
+ providerThread: harness.providerThread,
3894
+ now: yield* DateTime.now,
3895
+ text: "Continue.",
3896
+ attemptId: RunAttemptId.make(`attempt-${scenario}`),
3897
+ }),
3898
+ );
3899
+ yield* Deferred.await(retryShown);
3900
+ yield* TestClock.adjust("9 minutes");
3901
+ assert.isEmpty(harness.terminalEvents());
3902
+ yield* TestClock.adjust("1 minute");
3903
+ yield* harness.firstTerminal;
3904
+ yield* Deferred.await(interruptSent);
3905
+ const terminal = harness.terminalEvents()[0];
3906
+ assert.equal(terminal?.status, "failed");
3907
+ if (terminal?.status !== "failed") return;
3908
+ assert.equal(terminal.failure.class, "usage_limit");
3909
+ assert.equal(terminal.failure.code, "responseStreamDisconnected");
3910
+ assert.equal(
3911
+ terminal.failure.message,
3912
+ "exceeded retry limit, last status: 429 Too Many Requests",
3913
+ );
3914
+ assert.equal(terminal.retry?.attempt, 1);
3915
+ assert.equal(terminal.threadDisposition, "reusable");
3916
+ }).pipe(
3917
+ Effect.provide(
3918
+ Layer.mergeAll(
3919
+ IdAllocator.layer,
3920
+ McpProviderSessions.layer,
3921
+ NodeServices.layer,
3922
+ NodeCrypto.layer,
3923
+ ),
3924
+ ),
3925
+ ),
3926
+ ),
3927
+ );
3928
+
3929
it.effect.each(["completed", "interrupted"] as const)(
3930
"retains Codex reasoning parts when the turn is %s",
3931
(terminalStatus) =>
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
+73
@@ -158,6 +158,8 @@ import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"
158
159
const CODEX_PROVIDER = ProviderDriverKind.make("codex");
160
const CODEX_CAPACITY_MAX_RETRIES = 5;
161
+// Codex can announce a retry and never resume. Recovered retries resume model output within ~75s.
162
+const CODEX_PROVIDER_RETRY_STALL_TIMEOUT = "10 minutes";
163
export const CODEX_DRIVER_KIND = CODEX_PROVIDER;
164
export const CODEX_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(CODEX_DRIVER_KIND);
165
@@ -1141,6 +1143,17 @@ function codexExhaustionHttpStatus(errorInfo: CodexErrorInfo | null | undefined)
1143
: null;
1144
}
1145
1146
+function codexErrorHttpStatus(errorInfo: CodexErrorInfo | null | undefined): number | null {
1147
+ const detail =
1148
+ typeof errorInfo === "object" && errorInfo !== null ? Object.values(errorInfo)[0] : undefined;
1149
+ return typeof detail === "object" &&
1150
+ detail !== null &&
1151
+ "httpStatusCode" in detail &&
1152
+ typeof detail.httpStatusCode === "number"
1153
+ ? detail.httpStatusCode
1154
+ : null;
1155
+}
1156
+
1157
function codexUsageLimitError(
1158
code: string | null,
1159
errorInfo: CodexErrorInfo | null | undefined,
@@ -4899,6 +4912,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
4912
updated.set(context.providerTurnId, state);
4913
return updated;
4914
});
4915
+ yield* failStalledProviderRetry(context, state, payload.error.codexErrorInfo).pipe(
4916
+ Effect.delay(CODEX_PROVIDER_RETRY_STALL_TIMEOUT),
4917
+ Effect.forkIn(scope, { startImmediately: true }),
4918
+ );
4919
yield* emitProviderEvent({
4920
type: "turn_item.updated",
4921
driver: CODEX_PROVIDER,
@@ -6311,6 +6328,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6328
readonly failureMessage?: string;
6329
readonly failureErrorInfo?: CodexErrorInfo;
6330
readonly continueRun?: boolean;
6331
+ readonly stalledRetry?: {
6332
+ readonly state: ActiveCodexProviderRetry;
6333
+ readonly failure: NonNullable<ActiveCodexTurnContext["latestProviderFailure"]>;
6334
+ };
6335
}) =>
6336
// Capacity continuation already holds the permit across finalization and restart.
6337
(input.continueRun ? Function.identity : turnTerminalizationPermit.withPermits(1))(
@@ -6319,6 +6340,16 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6340
if (current !== input.context) {
6341
return false;
6342
}
6343
+ if (input.stalledRetry !== undefined) {
6344
+ // Checked under the permit so item/started cannot clear the retry mid-finalize.
6345
+ const retries = yield* Ref.get(providerRetries);
6346
+ if (
6347
+ retries.get(input.context.providerTurnId) !== input.stalledRetry.state ||
6348
+ (yield* Ref.get(interruptingNativeTurns)).has(input.nativeTurnId)
6349
+ )
6350
+ return false;
6351
+ input.context.latestProviderFailure ??= input.stalledRetry.failure;
6352
+ }
6353
input.context.fileChangeKindsByItem.clear();
6354
const providerRetry = yield* Ref.modify(providerRetries, (current) => {
6355
const retry = current.get(input.context.providerTurnId);
@@ -6637,6 +6668,48 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6668
}),
6669
);
6670
6671
+ // A retry that ends on 429 is the same exhaustion responseTooManyFailedAttempts reports.
6672
+ const failStalledProviderRetry = Effect.fn("CodexAdapterV2.failStalledProviderRetry")(
6673
+ function* (
6674
+ context: ActiveCodexTurnContext,
6675
+ state: ActiveCodexProviderRetry,
6676
+ errorInfo: CodexErrorInfo | null | undefined,
6677
+ ) {
6678
+ const finalized = yield* finalizeCodexTurn({
6679
+ context,
6680
+ nativeTurnId: context.nativeTurnId,
6681
+ status: "failed",
6682
+ completedAt: yield* DateTime.now,
6683
+ stalledRetry: {
6684
+ state,
6685
+ failure: {
6686
+ nativeMessage: state.nativeMessage,
6687
+ nativeCode: state.nativeCode,
6688
+ failure: {
6689
+ ...state.failure,
6690
+ retryable: false,
6691
+ ...(codexErrorHttpStatus(errorInfo) === 429 ? { class: "usage_limit" } : {}),
6692
+ },
6693
+ },
6694
+ },
6695
+ });
6696
+ if (!finalized) return;
6697
+ const nativeThreadId = yield* getNativeThreadId(context.providerThread);
6698
+ yield* client
6699
+ .request("turn/interrupt", { threadId: nativeThreadId, turnId: context.nativeTurnId })
6700
+ .pipe(
6701
+ Effect.timeout("10 seconds"),
6702
+ Effect.catch((cause) =>
6703
+ Effect.logWarning("orchestration-v2.codex-stalled-retry-interrupt-failed", {
6704
+ nativeThreadId,
6705
+ nativeTurnId: context.nativeTurnId,
6706
+ cause,
6707
+ }),
6708
+ ),
6709
+ );
6710
+ },
6711
+ );
6712
+
6713
// Codex can end an overloaded turn without retry advice. Continue its
6714
// persisted history with empty input instead of resending the user message.
6715
const retryCapacityFailure = Effect.fn("CodexAdapterV2.retryCapacityFailure")(function* (