@devries-cameron20 / t3code Mirror · read-only / commits / e21dff1988
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 23:15 UTC

fix(connect): preserve rotated credentials during shutdown (#19499)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

James C committed Oct 20, 2026 at 06:00 UTC e21dff1988bbba512dbb47f817944d9f44b6a281
4 files changed +335 -7
apps/server/src/cloud/CliTokenManager.test.ts
+203 -1
@@ -14,6 +14,7 @@ import * as Schema from "effect/Schema";
14 import * as Terminal from "effect/Terminal";
15 import * as TestClock from "effect/testing/TestClock";
16 import * as HttpClient from "effect/http/HttpClient";
17 +import * as HttpClientError from "effect/http/HttpClientError";
18 import * as HttpClientResponse from "effect/http/HttpClientResponse";
19
20 import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
@@ -184,6 +185,205 @@ const expiredStoredToken = new TextEncoder().encode(
185 JSON.stringify({ accessToken: "expired", refreshToken: "refresh-token-1", expiresAtEpochMs: 0 }),
186 );
187
188 +const provideRefreshManager = (
189 + httpClient: HttpClient.HttpClient,
190 + secrets = Layer.mock(ServerSecretStore.ServerSecretStore)({
191 + get: () => Effect.succeedSome(expiredStoredToken),
192 + }),
193 +) =>
194 + Effect.provide(
195 + CliTokenManager.layer.pipe(
196 + Layer.provide(
197 + Layer.mergeAll(
198 + Layer.succeed(HttpClient.HttpClient, httpClient),
199 + secrets,
200 + Layer.mock(ExternalLauncher.ExternalLauncher)({}),
201 + NodeServices.layer,
202 + ),
203 + ),
204 + Layer.provideMerge(ConfigProvider.layer(ConfigProvider.fromEnv({ env: TEST_ENV }))),
205 + ),
206 + );
207 +
208 +it.effect(
209 + "persists rotated credentials under the lock despite shutdown, while queued callers can cancel",
210 + () =>
211 + Effect.gen(function* () {
212 + const requestStarted = yield* Deferred.make<void>();
213 + const releaseRequest = yield* Deferred.make<void>();
214 + const persistStarted = yield* Deferred.make<void>();
215 + const releasePersist = yield* Deferred.make<void>();
216 + let stored = expiredStoredToken;
217 + let reads = 0;
218 + const httpClient = HttpClient.make((request) =>
219 + Effect.gen(function* () {
220 + yield* Deferred.succeed(requestStarted, undefined);
221 + yield* Deferred.await(releaseRequest);
222 + return HttpClientResponse.fromWeb(
223 + request,
224 + Response.json({
225 + access_token: "rotated-access",
226 + refresh_token: "rotated-refresh",
227 + expires_in: 3600,
228 + token_type: "bearer",
229 + }),
230 + );
231 + }),
232 + );
233 + const secrets = Layer.mock(ServerSecretStore.ServerSecretStore)({
234 + get: () =>
235 + Effect.sync(() => {
236 + reads++;
237 + return Option.some(stored);
238 + }),
239 + set: (_name, value) =>
240 + Effect.gen(function* () {
241 + yield* Deferred.succeed(persistStarted, undefined);
242 + yield* Deferred.await(releasePersist);
243 + stored = new Uint8Array(value);
244 + }),
245 + });
246 +
247 + yield* Effect.gen(function* () {
248 + const manager = yield* CliTokenManager.CloudCliTokenManager;
249 + const refresh = yield* manager.getExisting.pipe(Effect.forkChild);
250 + assert.isTrue(
251 + yield* Effect.raceFirst(
252 + Deferred.await(requestStarted).pipe(Effect.as(true)),
253 + Fiber.await(refresh).pipe(Effect.as(false)),
254 + ),
255 + "refresh failed before reaching the endpoint",
256 + );
257 + const waiter = yield* manager.getExisting.pipe(
258 + Effect.forkChild({ startImmediately: true }),
259 + );
260 + yield* Fiber.interrupt(waiter);
261 + assert.equal(reads, 1, "queued caller must not read credentials during refresh");
262 + const interruption = yield* Fiber.interrupt(refresh).pipe(
263 + Effect.forkChild({ startImmediately: true }),
264 + );
265 + yield* Deferred.succeed(releaseRequest, undefined);
266 + const reachedPersist = yield* Effect.raceFirst(
267 + Deferred.await(persistStarted).pipe(Effect.as(true)),
268 + Fiber.await(refresh).pipe(Effect.as(false)),
269 + );
270 + assert.isTrue(reachedPersist, "shutdown discarded the rotated credential");
271 + const reader = yield* manager.getExisting.pipe(
272 + Effect.forkChild({ startImmediately: true }),
273 + );
274 + assert.equal(reads, 1, "credential lock must cover persistence");
275 + yield* Deferred.succeed(releasePersist, undefined);
276 + yield* Fiber.join(interruption);
277 + const token = Option.getOrThrow(yield* Fiber.join(reader));
278 + assert.equal(token.refreshToken, "rotated-refresh");
279 + assert.equal(token.accessToken, "rotated-access");
280 + }).pipe(
281 + Effect.ensuring(Deferred.succeed(releaseRequest, undefined)),
282 + Effect.ensuring(Deferred.succeed(releasePersist, undefined)),
283 + provideRefreshManager(httpClient, secrets),
284 + );
285 + }),
286 +);
287 +
288 +it.effect.each([
289 + { status: 400, code: "invalid_grant", requiresLogin: true, diagnostic: true },
290 + { status: 403, code: "private-provider-secret", requiresLogin: true, diagnostic: false },
291 + { status: 408, code: "invalid_grant", requiresLogin: false, diagnostic: false },
292 + { status: 429, code: "invalid_grant", requiresLogin: false, diagnostic: false },
293 + { status: 500, code: "invalid_grant", requiresLogin: false, diagnostic: false },
294 +])(
295 + "refresh HTTP $status exposes only a safe login diagnostic",
296 + ({ status, code, requiresLogin, diagnostic }) =>
297 + Effect.gen(function* () {
298 + const manager = yield* CliTokenManager.CloudCliTokenManager;
299 + const error = yield* manager.getExisting.pipe(Effect.flip);
300 + assert.instanceOf(error, CliTokenManager.CloudCliCredentialRefreshError);
301 + assert.equal(error.requiresLogin, requiresLogin);
302 + assert.equal(error.message.includes("t3 connect login"), requiresLogin);
303 + assert.equal(error.message.includes("OAuth error: invalid_grant"), diagnostic);
304 + assert.notInclude(error.message, "private");
305 + assert.equal(error.message.includes("OAuth error:"), diagnostic);
306 + assert.isTrue(yield* manager.hasCredential);
307 + }).pipe(
308 + provideRefreshManager(
309 + HttpClient.make((request) =>
310 + Effect.succeed(
311 + HttpClientResponse.fromWeb(
312 + request,
313 + Response.json(
314 + {
315 + error: code,
316 + error_description: "private-description",
317 + refresh_token: "private-refresh-token",
318 + },
319 + { status },
320 + ),
321 + ),
322 + ),
323 + ),
324 + ),
325 + ),
326 +);
327 +
328 +it.effect.each(["malformed", "unreadable", "stalled"] as const)(
329 + "a $0 rejection body preserves HTTP status and its underlying failure",
330 + (bodyKind) =>
331 + Effect.gen(function* () {
332 + const readerFailure = new Error("private-reader-failure");
333 + const client = HttpClient.make((request) =>
334 + Effect.succeed(
335 + HttpClientResponse.fromWeb(
336 + request,
337 + new Response(
338 + bodyKind === "malformed"
339 + ? "{"
340 + : new ReadableStream({
341 + start: (controller) => {
342 + if (bodyKind === "unreadable") controller.error(readerFailure);
343 + else controller.enqueue(new TextEncoder().encode("{"));
344 + },
345 + }),
346 + { status: 400 },
347 + ),
348 + ),
349 + ).pipe(Effect.delay(bodyKind === "stalled" ? Duration.seconds(4) : Duration.zero)),
350 + );
351 + yield* Effect.gen(function* () {
352 + const manager = yield* CliTokenManager.CloudCliTokenManager;
353 + const refresh = yield* manager.getExisting.pipe(
354 + Effect.flip,
355 + Effect.timed,
356 + Effect.forkChild,
357 + );
358 + const reader = yield* manager.hasCredential.pipe(Effect.forkChild);
359 + if (bodyKind === "stalled") yield* TestClock.adjust(Duration.seconds(14));
360 + const [duration, error] = yield* Fiber.join(refresh);
361 + if (bodyKind === "stalled") assert.equal(Duration.toMillis(duration), 10_000);
362 + assert.instanceOf(error, CliTokenManager.CloudCliCredentialRefreshError);
363 + assert.isTrue(error.requiresLogin);
364 + assert.include(error.message, "t3 connect login");
365 + assert.notInclude(error.message, "private");
366 + assert.notInclude(error.message, "OAuth error:");
367 + assert.isTrue(HttpClientError.isHttpClientError(error.cause));
368 + if (!HttpClientError.isHttpClientError(error.cause)) return;
369 + assert.equal(error.cause.reason._tag, "StatusCodeError");
370 + if (error.cause.reason._tag !== "StatusCodeError") return;
371 + assert.equal(error.cause.reason.response.status, 400);
372 + const underlying = error.cause.reason.cause;
373 + if (bodyKind === "unreadable") {
374 + assert.isTrue(HttpClientError.isHttpClientError(underlying));
375 + if (!HttpClientError.isHttpClientError(underlying)) return;
376 + assert.equal(underlying.reason._tag, "DecodeError");
377 + if (underlying.reason._tag === "DecodeError")
378 + assert.strictEqual(underlying.reason.cause, readerFailure);
379 + } else if (bodyKind === "stalled") {
380 + assert.isTrue(Cause.isTimeoutError(underlying));
381 + }
382 + assert.isTrue(yield* Fiber.join(reader));
383 + }).pipe(provideRefreshManager(client));
384 + }),
385 +);
386 +
387 it.effect("a stalled token refresh fails and releases the credential lock", () =>
388 Effect.gen(function* () {
389 const manager = yield* CliTokenManager.CloudCliTokenManager;
@@ -192,7 +392,9 @@ it.effect("a stalled token refresh fails and releases the credential lock", () =
392
393 yield* TestClock.adjust(Duration.seconds(10));
394
195 - assert.instanceOf(yield* Fiber.join(refresh), CliTokenManager.CloudCliCredentialRefreshError);
395 + const error = yield* Fiber.join(refresh);
396 + assert.instanceOf(error, CliTokenManager.CloudCliCredentialRefreshError);
397 + assert.isFalse(error.requiresLogin);
398 assert.isTrue(yield* Fiber.join(hasCredential));
399 }).pipe(
400 Effect.provide(
apps/server/src/cloud/CliTokenManager.ts
+88 -4
@@ -146,6 +146,21 @@ const OAuthErrorResponse = Schema.Struct({
146 error: Schema.String,
147 error_description: Schema.optional(Schema.String),
148 });
149 +const OAuthRefreshErrorCode = Schema.Literals([
150 + "invalid_grant",
151 + "invalid_client",
152 + "unauthorized_client",
153 + "unsupported_grant_type",
154 + "invalid_request",
155 + "invalid_scope",
156 + "access_denied",
157 +]);
158 +const decodeOAuthRefreshErrorResponse = Schema.decodeUnknownOption(
159 + Schema.Struct({ error: OAuthRefreshErrorCode }),
160 +);
161 +
162 +const isPermanentRefreshStatus = (status: number): boolean =>
163 + status >= 400 && status < 500 && status !== 408 && status !== 429;
164
165 const DeviceAuthorizationResponse = Schema.Struct({
166 device_code: Schema.String,
@@ -190,13 +205,27 @@ export class CloudCliCredentialRemovalError extends Schema.TaggedError<CloudCliC
205
206 export class CloudCliCredentialRefreshError extends Schema.TaggedError<CloudCliCredentialRefreshError>()(
207 "CloudCliCredentialRefreshError",
193 - { cause: Schema.Defect() },
208 + {
209 + cause: Schema.Defect(),
210 + status: Schema.optional(Schema.Number),
211 + oauthErrorCode: Schema.optional(OAuthRefreshErrorCode),
212 + },
213 ) {
214 + get requiresLogin(): boolean {
215 + return this.status !== undefined && isPermanentRefreshStatus(this.status);
216 + }
217 +
218 override get message(): string {
196 - return "Could not refresh the T3 Connect CLI credential.";
219 + return this.requiresLogin
220 + ? `Could not refresh the T3 Connect CLI credential${
221 + this.oauthErrorCode === undefined ? "" : ` (OAuth error: ${this.oauthErrorCode})`
222 + }. Run \`t3 connect login\` to authorize again.`
223 + : "Could not refresh the T3 Connect CLI credential.";
224 }
225 }
226
227 +export const isCloudCliCredentialRefreshError = Schema.is(CloudCliCredentialRefreshError);
228 +
229 export class CloudCliCredentialReadError extends Schema.TaggedError<CloudCliCredentialReadError>()(
230 "CloudCliCredentialReadError",
231 { cause: Schema.Defect() },
@@ -289,11 +318,52 @@ const exchangeToken = Effect.fn("cloud.cli_token.exchange")(function* (
318 params: Record<string, string>,
319 ) {
320 const httpClient = (yield* HttpClient.HttpClient).pipe(HttpClient.filterStatusOk);
321 + const startedAt = yield* Clock.monotonicTimeNanos;
322 return yield* HttpClientRequest.post(metadata.tokenEndpoint).pipe(
323 HttpClientRequest.bodyUrlParams(params),
324 httpClient.execute,
325 Effect.flatMap((response) => readTokenResponse(response, params)),
326 Effect.timeout(OAUTH_REQUEST_TIMEOUT),
327 + Effect.catchTags({
328 + HttpClientError: (cause) =>
329 + Effect.gen(function* () {
330 + const reason = cause.reason;
331 + if (
332 + params.grant_type !== "refresh_token" ||
333 + reason._tag !== "StatusCodeError" ||
334 + !isPermanentRefreshStatus(reason.response.status)
335 + ) {
336 + return yield* cause;
337 + }
338 + // Reading a rejection body shares the exchange budget without losing
339 + // the known status when the body stalls or cannot be decoded.
340 + const elapsed = Duration.toMillis(
341 + Duration.nanos((yield* Clock.monotonicTimeNanos) - startedAt),
342 + );
343 + const body = yield* reason.response.json.pipe(
344 + Effect.timeout(Math.max(0, Duration.toMillis(OAUTH_REQUEST_TIMEOUT) - elapsed)),
345 + Effect.mapError(
346 + (cause) =>
347 + new CloudCliCredentialRefreshError({
348 + status: reason.response.status,
349 + cause: new HttpClientError.HttpClientError({
350 + reason: new HttpClientError.StatusCodeError({
351 + request: reason.request,
352 + response: reason.response,
353 + cause,
354 + }),
355 + }),
356 + }),
357 + ),
358 + );
359 + const diagnostic = decodeOAuthRefreshErrorResponse(body);
360 + return yield* new CloudCliCredentialRefreshError({
361 + cause,
362 + status: reason.response.status,
363 + ...(Option.isSome(diagnostic) ? { oauthErrorCode: diagnostic.value.error } : {}),
364 + });
365 + }),
366 + }),
367 );
368 });
369
@@ -516,12 +586,26 @@ export const make = Effect.gen(function* () {
586 if (token.value.expiresAtEpochMs - CLOUD_CLI_OAUTH_REFRESH_EARLY_MS > now) {
587 return token;
588 }
519 - return Option.some(yield* refresh(token.value).pipe(Effect.flatMap(persist)));
589 + return Option.some(
590 + // The endpoint can rotate the refresh token before shutdown interrupts us.
591 + // Keep the lock through persistence; the exchange still times out internally.
592 + yield* refresh(token.value).pipe(Effect.flatMap(persist), Effect.uninterruptible),
593 + );
594 });
595
596 const getExisting = semaphore.withPermits(1)(
597 getExistingNoLock().pipe(
524 - Effect.mapError((cause) => new CloudCliCredentialRefreshError({ cause })),
598 + Effect.mapError((cause) =>
599 + isCloudCliCredentialRefreshError(cause)
600 + ? cause
601 + : new CloudCliCredentialRefreshError({
602 + cause,
603 + ...(HttpClientError.isHttpClientError(cause) &&
604 + cause.reason._tag === "StatusCodeError"
605 + ? { status: cause.reason.response.status }
606 + : {}),
607 + }),
608 + ),
609 Effect.provide(services),
610 ),
611 );
apps/server/src/cloud/CloudLink.lifecycle.test.ts
+41 -1
@@ -242,6 +242,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
242 readonly respond?: () => Response;
243 readonly address?: HttpServer.HttpServer["Service"]["address"];
244 readonly respondEffect?: Effect.Effect<Response>;
245 + readonly getExisting?: CliTokenManager.CloudCliTokenManager["Service"]["getExisting"];
246 }
247
248 const writeLauncherState = (update: ServiceUpdateRecord) =>
@@ -318,7 +319,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
319 CliTokenManager.CloudCliTokenManager,
320 CliTokenManager.CloudCliTokenManager.of({
321 get: unusedSecretStoreOperation(),
321 - getExisting: Effect.succeedSome(cliToken),
322 + getExisting: harness.getExisting ?? Effect.succeedSome(cliToken),
323 hasCredential: unusedSecretStoreOperation(),
324 store: () => unusedSecretStoreOperation(),
325 clear: unusedSecretStoreOperation(),
@@ -1337,6 +1338,45 @@ describe("releaseManagedTunnelOnShutdown", () => {
1338 );
1339 });
1340
1341 + it.effect("startup retries transient refresh failures but stops when login is required", () => {
1342 + const { store } = makeMemorySecretStore(managedLinkSecrets);
1343 + const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1344 + const applyConfigCalls: Array<unknown> = [];
1345 + const permanent = new CliTokenManager.CloudCliCredentialRefreshError({
1346 + cause: new Error("private-oauth-failure"),
1347 + status: 400,
1348 + oauthErrorCode: "invalid_grant",
1349 + });
1350 + let attempts = 0;
1351 + const getExisting = Effect.suspend(() => {
1352 + attempts++;
1353 + if (attempts > 2) return Effect.die("retried a permanent credential rejection");
1354 + return Effect.fail(
1355 + attempts === 2
1356 + ? permanent
1357 + : new CliTokenManager.CloudCliCredentialRefreshError({
1358 + cause: new Error("connection reset"),
1359 + }),
1360 + );
1361 + });
1362 +
1363 + return Effect.gen(function* () {
1364 + const link = yield* CloudLink.CloudLink;
1365 + const registration = yield* retryManagedTunnelRegistration(
1366 + link.reconcileDesiredLinkIfStillDesired("http://127.0.0.1:3773"),
1367 + CloudLink.shouldRetryCloudLink,
1368 + ).pipe(Effect.flip, Effect.forkChild);
1369 + yield* TestClock.adjust("10 seconds");
1370 + const error = yield* Fiber.join(registration);
1371 + expect(attempts).toBe(2);
1372 + expect(error).toBe(permanent);
1373 + expect(error.message).toContain("t3 connect login");
1374 + expect(error.message).not.toContain("private");
1375 + expect(requests).toEqual([]);
1376 + expect(applyConfigCalls).toEqual([]);
1377 + }).pipe(provideReleaseHarness({ store, requests, applyConfigCalls, getExisting }));
1378 + });
1379 +
1380 it.effect("keeps a tunnel configuration replaced during recovery", () => {
1381 const { store, values } = makeMemorySecretStore([
1382 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"],
apps/server/src/cloud/CloudLink.ts
+3 -1
@@ -291,7 +291,9 @@ const isPermanentLinkError = Schema.is(
291 );
292
293 export const shouldRetryCloudLink = (error: unknown): boolean =>
294 - shouldRetryRelayRequest(error) && !isPermanentLinkError(error);
294 + shouldRetryRelayRequest(error) &&
295 + !isPermanentLinkError(error) &&
296 + !(CliTokenManager.isCloudCliCredentialRefreshError(error) && error.requiresLogin);
297
298 const rollbackFailed = (cause: unknown) =>
299 Effect.logWarning("Could not roll back a T3 Connect preference", { cause });