@devries-cameron20 / t3code Mirror · read-only / commits / 362ac8d6d5
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 22:15 UTC

fix(server): fail Codex turns stuck in a provider retry (#33130)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

maria committed Nov 11, 2026 at 19:58 UTC 362ac8d6d5ced16089b98b30f4638e7577ca6581
2 files changed +173
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts
+100
@@ -3826,6 +3826,106 @@ describe("CodexAdapterV2 post-settle continuation", () => {
3826 ),
3827 );
3828
3829 + it.effect("fails a stalled 429 provider retry as a usage limit and stops the Codex turn", () =>
3830 + Effect.scoped(
3831 + Effect.gen(function* () {
3832 + const scenario = "codex-provider-retry-stall";
3833 + const nativeThreadId = `native-${scenario}-thread`;
3834 + const nativeTurnId = `native-${scenario}-turn`;
3835 + const transcript = makeCodexReplayTranscript({
3836 + scenario,
3837 + entries: [
3838 + ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt: "Continue." }),
3839 + {
3840 + type: "emit_inbound",
3841 + label: "error/retry",
3842 + frame: {
3843 + method: "error",
3844 + params: {
3845 + threadId: nativeThreadId,
3846 + turnId: nativeTurnId,
3847 + willRetry: true,
3848 + error: {
3849 + message: "Reconnecting... 1/5",
3850 + additionalDetails: "exceeded retry limit, last status: 429 Too Many Requests",
3851 + codexErrorInfo: { responseStreamDisconnected: { httpStatusCode: 429 } },
3852 + },
3853 + },
3854 + },
3855 + },
3856 + {
3857 + type: "expect_outbound",
3858 + label: "turn/interrupt",
3859 + frame: {
3860 + id: 4,
3861 + method: "turn/interrupt",
3862 + params: { threadId: nativeThreadId, turnId: nativeTurnId },
3863 + },
3864 + },
3865 + { type: "emit_inbound", label: "turn/interrupt", frame: { id: 4, result: {} } },
3866 + {
3867 + type: "emit_inbound",
3868 + label: "turn/completed",
3869 + frame: {
3870 + method: "turn/completed",
3871 + params: {
3872 + threadId: nativeThreadId,
3873 + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "interrupted" }),
3874 + },
3875 + },
3876 + },
3877 + ],
3878 + });
3879 + const retryShown = yield* Deferred.make<void>();
3880 + const interruptSent = yield* Deferred.make<void>();
3881 + const harness = yield* makeCodexReplayHarness(
3882 + transcript,
3883 + (event) =>
3884 + event.type === "turn_item.updated" && event.turnItem.type === "error"
3885 + ? Deferred.succeed(retryShown, undefined)
3886 + : Effect.void,
3887 + (method) =>
3888 + method === "turn/interrupt" ? Deferred.succeed(interruptSent, undefined) : Effect.void,
3889 + );
3890 + yield* harness.runtime.startTurn(
3891 + makeCodexTestTurnInput({
3892 + threadId: harness.threadId,
3893 + providerThread: harness.providerThread,
3894 + now: yield* DateTime.now,
3895 + text: "Continue.",
3896 + attemptId: RunAttemptId.make(`attempt-${scenario}`),
3897 + }),
3898 + );
3899 + yield* Deferred.await(retryShown);
3900 + yield* TestClock.adjust("9 minutes");
3901 + assert.isEmpty(harness.terminalEvents());
3902 + yield* TestClock.adjust("1 minute");
3903 + yield* harness.firstTerminal;
3904 + yield* Deferred.await(interruptSent);
3905 + const terminal = harness.terminalEvents()[0];
3906 + assert.equal(terminal?.status, "failed");
3907 + if (terminal?.status !== "failed") return;
3908 + assert.equal(terminal.failure.class, "usage_limit");
3909 + assert.equal(terminal.failure.code, "responseStreamDisconnected");
3910 + assert.equal(
3911 + terminal.failure.message,
3912 + "exceeded retry limit, last status: 429 Too Many Requests",
3913 + );
3914 + assert.equal(terminal.retry?.attempt, 1);
3915 + assert.equal(terminal.threadDisposition, "reusable");
3916 + }).pipe(
3917 + Effect.provide(
3918 + Layer.mergeAll(
3919 + IdAllocator.layer,
3920 + McpProviderSessions.layer,
3921 + NodeServices.layer,
3922 + NodeCrypto.layer,
3923 + ),
3924 + ),
3925 + ),
3926 + ),
3927 + );
3928 +
3929 it.effect.each(["completed", "interrupted"] as const)(
3930 "retains Codex reasoning parts when the turn is %s",
3931 (terminalStatus) =>
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
+73
@@ -158,6 +158,8 @@ import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"
158
159 const CODEX_PROVIDER = ProviderDriverKind.make("codex");
160 const CODEX_CAPACITY_MAX_RETRIES = 5;
161 +// Codex can announce a retry and never resume. Recovered retries resume model output within ~75s.
162 +const CODEX_PROVIDER_RETRY_STALL_TIMEOUT = "10 minutes";
163 export const CODEX_DRIVER_KIND = CODEX_PROVIDER;
164 export const CODEX_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(CODEX_DRIVER_KIND);
165
@@ -1141,6 +1143,17 @@ function codexExhaustionHttpStatus(errorInfo: CodexErrorInfo | null | undefined)
1143 : null;
1144 }
1145
1146 +function codexErrorHttpStatus(errorInfo: CodexErrorInfo | null | undefined): number | null {
1147 + const detail =
1148 + typeof errorInfo === "object" && errorInfo !== null ? Object.values(errorInfo)[0] : undefined;
1149 + return typeof detail === "object" &&
1150 + detail !== null &&
1151 + "httpStatusCode" in detail &&
1152 + typeof detail.httpStatusCode === "number"
1153 + ? detail.httpStatusCode
1154 + : null;
1155 +}
1156 +
1157 function codexUsageLimitError(
1158 code: string | null,
1159 errorInfo: CodexErrorInfo | null | undefined,
@@ -4899,6 +4912,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
4912 updated.set(context.providerTurnId, state);
4913 return updated;
4914 });
4915 + yield* failStalledProviderRetry(context, state, payload.error.codexErrorInfo).pipe(
4916 + Effect.delay(CODEX_PROVIDER_RETRY_STALL_TIMEOUT),
4917 + Effect.forkIn(scope, { startImmediately: true }),
4918 + );
4919 yield* emitProviderEvent({
4920 type: "turn_item.updated",
4921 driver: CODEX_PROVIDER,
@@ -6311,6 +6328,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6328 readonly failureMessage?: string;
6329 readonly failureErrorInfo?: CodexErrorInfo;
6330 readonly continueRun?: boolean;
6331 + readonly stalledRetry?: {
6332 + readonly state: ActiveCodexProviderRetry;
6333 + readonly failure: NonNullable<ActiveCodexTurnContext["latestProviderFailure"]>;
6334 + };
6335 }) =>
6336 // Capacity continuation already holds the permit across finalization and restart.
6337 (input.continueRun ? Function.identity : turnTerminalizationPermit.withPermits(1))(
@@ -6319,6 +6340,16 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6340 if (current !== input.context) {
6341 return false;
6342 }
6343 + if (input.stalledRetry !== undefined) {
6344 + // Checked under the permit so item/started cannot clear the retry mid-finalize.
6345 + const retries = yield* Ref.get(providerRetries);
6346 + if (
6347 + retries.get(input.context.providerTurnId) !== input.stalledRetry.state ||
6348 + (yield* Ref.get(interruptingNativeTurns)).has(input.nativeTurnId)
6349 + )
6350 + return false;
6351 + input.context.latestProviderFailure ??= input.stalledRetry.failure;
6352 + }
6353 input.context.fileChangeKindsByItem.clear();
6354 const providerRetry = yield* Ref.modify(providerRetries, (current) => {
6355 const retry = current.get(input.context.providerTurnId);
@@ -6637,6 +6668,48 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6668 }),
6669 );
6670
6671 + // A retry that ends on 429 is the same exhaustion responseTooManyFailedAttempts reports.
6672 + const failStalledProviderRetry = Effect.fn("CodexAdapterV2.failStalledProviderRetry")(
6673 + function* (
6674 + context: ActiveCodexTurnContext,
6675 + state: ActiveCodexProviderRetry,
6676 + errorInfo: CodexErrorInfo | null | undefined,
6677 + ) {
6678 + const finalized = yield* finalizeCodexTurn({
6679 + context,
6680 + nativeTurnId: context.nativeTurnId,
6681 + status: "failed",
6682 + completedAt: yield* DateTime.now,
6683 + stalledRetry: {
6684 + state,
6685 + failure: {
6686 + nativeMessage: state.nativeMessage,
6687 + nativeCode: state.nativeCode,
6688 + failure: {
6689 + ...state.failure,
6690 + retryable: false,
6691 + ...(codexErrorHttpStatus(errorInfo) === 429 ? { class: "usage_limit" } : {}),
6692 + },
6693 + },
6694 + },
6695 + });
6696 + if (!finalized) return;
6697 + const nativeThreadId = yield* getNativeThreadId(context.providerThread);
6698 + yield* client
6699 + .request("turn/interrupt", { threadId: nativeThreadId, turnId: context.nativeTurnId })
6700 + .pipe(
6701 + Effect.timeout("10 seconds"),
6702 + Effect.catch((cause) =>
6703 + Effect.logWarning("orchestration-v2.codex-stalled-retry-interrupt-failed", {
6704 + nativeThreadId,
6705 + nativeTurnId: context.nativeTurnId,
6706 + cause,
6707 + }),
6708 + ),
6709 + );
6710 + },
6711 + );
6712 +
6713 // Codex can end an overloaded turn without retry advice. Continue its
6714 // persisted history with empty input instead of resending the user message.
6715 const retryCapacityFailure = Effect.fn("CodexAdapterV2.retryCapacityFailure")(function* (