The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 23:15 UTC
fix(connect): preserve rotated credentials during shutdown (#19499)
Co-authored-by: Julius Marminge <julius0216@outlook.com>
James C committed
Oct 20, 2026 at 06:00 UTC
e21dff1988bbba512dbb47f817944d9f44b6a281
4 files changed
+335
-7
apps/server/src/cloud/CliTokenManager.test.ts
+203
-1
@@ -14,6 +14,7 @@ import * as Schema from "effect/Schema";
14
import * as Terminal from "effect/Terminal";
15
import * as TestClock from "effect/testing/TestClock";
16
import * as HttpClient from "effect/http/HttpClient";
17
+import * as HttpClientError from "effect/http/HttpClientError";
18
import * as HttpClientResponse from "effect/http/HttpClientResponse";
19
20
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
@@ -184,6 +185,205 @@ const expiredStoredToken = new TextEncoder().encode(
185
JSON.stringify({ accessToken: "expired", refreshToken: "refresh-token-1", expiresAtEpochMs: 0 }),
186
);
187
188
+const provideRefreshManager = (
189
+ httpClient: HttpClient.HttpClient,
190
+ secrets = Layer.mock(ServerSecretStore.ServerSecretStore)({
191
+ get: () => Effect.succeedSome(expiredStoredToken),
192
+ }),
193
+) =>
194
+ Effect.provide(
195
+ CliTokenManager.layer.pipe(
196
+ Layer.provide(
197
+ Layer.mergeAll(
198
+ Layer.succeed(HttpClient.HttpClient, httpClient),
199
+ secrets,
200
+ Layer.mock(ExternalLauncher.ExternalLauncher)({}),
201
+ NodeServices.layer,
202
+ ),
203
+ ),
204
+ Layer.provideMerge(ConfigProvider.layer(ConfigProvider.fromEnv({ env: TEST_ENV }))),
205
+ ),
206
+ );
207
+
208
+it.effect(
209
+ "persists rotated credentials under the lock despite shutdown, while queued callers can cancel",
210
+ () =>
211
+ Effect.gen(function* () {
212
+ const requestStarted = yield* Deferred.make<void>();
213
+ const releaseRequest = yield* Deferred.make<void>();
214
+ const persistStarted = yield* Deferred.make<void>();
215
+ const releasePersist = yield* Deferred.make<void>();
216
+ let stored = expiredStoredToken;
217
+ let reads = 0;
218
+ const httpClient = HttpClient.make((request) =>
219
+ Effect.gen(function* () {
220
+ yield* Deferred.succeed(requestStarted, undefined);
221
+ yield* Deferred.await(releaseRequest);
222
+ return HttpClientResponse.fromWeb(
223
+ request,
224
+ Response.json({
225
+ access_token: "rotated-access",
226
+ refresh_token: "rotated-refresh",
227
+ expires_in: 3600,
228
+ token_type: "bearer",
229
+ }),
230
+ );
231
+ }),
232
+ );
233
+ const secrets = Layer.mock(ServerSecretStore.ServerSecretStore)({
234
+ get: () =>
235
+ Effect.sync(() => {
236
+ reads++;
237
+ return Option.some(stored);
238
+ }),
239
+ set: (_name, value) =>
240
+ Effect.gen(function* () {
241
+ yield* Deferred.succeed(persistStarted, undefined);
242
+ yield* Deferred.await(releasePersist);
243
+ stored = new Uint8Array(value);
244
+ }),
245
+ });
246
+
247
+ yield* Effect.gen(function* () {
248
+ const manager = yield* CliTokenManager.CloudCliTokenManager;
249
+ const refresh = yield* manager.getExisting.pipe(Effect.forkChild);
250
+ assert.isTrue(
251
+ yield* Effect.raceFirst(
252
+ Deferred.await(requestStarted).pipe(Effect.as(true)),
253
+ Fiber.await(refresh).pipe(Effect.as(false)),
254
+ ),
255
+ "refresh failed before reaching the endpoint",
256
+ );
257
+ const waiter = yield* manager.getExisting.pipe(
258
+ Effect.forkChild({ startImmediately: true }),
259
+ );
260
+ yield* Fiber.interrupt(waiter);
261
+ assert.equal(reads, 1, "queued caller must not read credentials during refresh");
262
+ const interruption = yield* Fiber.interrupt(refresh).pipe(
263
+ Effect.forkChild({ startImmediately: true }),
264
+ );
265
+ yield* Deferred.succeed(releaseRequest, undefined);
266
+ const reachedPersist = yield* Effect.raceFirst(
267
+ Deferred.await(persistStarted).pipe(Effect.as(true)),
268
+ Fiber.await(refresh).pipe(Effect.as(false)),
269
+ );
270
+ assert.isTrue(reachedPersist, "shutdown discarded the rotated credential");
271
+ const reader = yield* manager.getExisting.pipe(
272
+ Effect.forkChild({ startImmediately: true }),
273
+ );
274
+ assert.equal(reads, 1, "credential lock must cover persistence");
275
+ yield* Deferred.succeed(releasePersist, undefined);
276
+ yield* Fiber.join(interruption);
277
+ const token = Option.getOrThrow(yield* Fiber.join(reader));
278
+ assert.equal(token.refreshToken, "rotated-refresh");
279
+ assert.equal(token.accessToken, "rotated-access");
280
+ }).pipe(
281
+ Effect.ensuring(Deferred.succeed(releaseRequest, undefined)),
282
+ Effect.ensuring(Deferred.succeed(releasePersist, undefined)),
283
+ provideRefreshManager(httpClient, secrets),
284
+ );
285
+ }),
286
+);
287
+
288
+it.effect.each([
289
+ { status: 400, code: "invalid_grant", requiresLogin: true, diagnostic: true },
290
+ { status: 403, code: "private-provider-secret", requiresLogin: true, diagnostic: false },
291
+ { status: 408, code: "invalid_grant", requiresLogin: false, diagnostic: false },
292
+ { status: 429, code: "invalid_grant", requiresLogin: false, diagnostic: false },
293
+ { status: 500, code: "invalid_grant", requiresLogin: false, diagnostic: false },
294
+])(
295
+ "refresh HTTP $status exposes only a safe login diagnostic",
296
+ ({ status, code, requiresLogin, diagnostic }) =>
297
+ Effect.gen(function* () {
298
+ const manager = yield* CliTokenManager.CloudCliTokenManager;
299
+ const error = yield* manager.getExisting.pipe(Effect.flip);
300
+ assert.instanceOf(error, CliTokenManager.CloudCliCredentialRefreshError);
301
+ assert.equal(error.requiresLogin, requiresLogin);
302
+ assert.equal(error.message.includes("t3 connect login"), requiresLogin);
303
+ assert.equal(error.message.includes("OAuth error: invalid_grant"), diagnostic);
304
+ assert.notInclude(error.message, "private");
305
+ assert.equal(error.message.includes("OAuth error:"), diagnostic);
306
+ assert.isTrue(yield* manager.hasCredential);
307
+ }).pipe(
308
+ provideRefreshManager(
309
+ HttpClient.make((request) =>
310
+ Effect.succeed(
311
+ HttpClientResponse.fromWeb(
312
+ request,
313
+ Response.json(
314
+ {
315
+ error: code,
316
+ error_description: "private-description",
317
+ refresh_token: "private-refresh-token",
318
+ },
319
+ { status },
320
+ ),
321
+ ),
322
+ ),
323
+ ),
324
+ ),
325
+ ),
326
+);
327
+
328
+it.effect.each(["malformed", "unreadable", "stalled"] as const)(
329
+ "a $0 rejection body preserves HTTP status and its underlying failure",
330
+ (bodyKind) =>
331
+ Effect.gen(function* () {
332
+ const readerFailure = new Error("private-reader-failure");
333
+ const client = HttpClient.make((request) =>
334
+ Effect.succeed(
335
+ HttpClientResponse.fromWeb(
336
+ request,
337
+ new Response(
338
+ bodyKind === "malformed"
339
+ ? "{"
340
+ : new ReadableStream({
341
+ start: (controller) => {
342
+ if (bodyKind === "unreadable") controller.error(readerFailure);
343
+ else controller.enqueue(new TextEncoder().encode("{"));
344
+ },
345
+ }),
346
+ { status: 400 },
347
+ ),
348
+ ),
349
+ ).pipe(Effect.delay(bodyKind === "stalled" ? Duration.seconds(4) : Duration.zero)),
350
+ );
351
+ yield* Effect.gen(function* () {
352
+ const manager = yield* CliTokenManager.CloudCliTokenManager;
353
+ const refresh = yield* manager.getExisting.pipe(
354
+ Effect.flip,
355
+ Effect.timed,
356
+ Effect.forkChild,
357
+ );
358
+ const reader = yield* manager.hasCredential.pipe(Effect.forkChild);
359
+ if (bodyKind === "stalled") yield* TestClock.adjust(Duration.seconds(14));
360
+ const [duration, error] = yield* Fiber.join(refresh);
361
+ if (bodyKind === "stalled") assert.equal(Duration.toMillis(duration), 10_000);
362
+ assert.instanceOf(error, CliTokenManager.CloudCliCredentialRefreshError);
363
+ assert.isTrue(error.requiresLogin);
364
+ assert.include(error.message, "t3 connect login");
365
+ assert.notInclude(error.message, "private");
366
+ assert.notInclude(error.message, "OAuth error:");
367
+ assert.isTrue(HttpClientError.isHttpClientError(error.cause));
368
+ if (!HttpClientError.isHttpClientError(error.cause)) return;
369
+ assert.equal(error.cause.reason._tag, "StatusCodeError");
370
+ if (error.cause.reason._tag !== "StatusCodeError") return;
371
+ assert.equal(error.cause.reason.response.status, 400);
372
+ const underlying = error.cause.reason.cause;
373
+ if (bodyKind === "unreadable") {
374
+ assert.isTrue(HttpClientError.isHttpClientError(underlying));
375
+ if (!HttpClientError.isHttpClientError(underlying)) return;
376
+ assert.equal(underlying.reason._tag, "DecodeError");
377
+ if (underlying.reason._tag === "DecodeError")
378
+ assert.strictEqual(underlying.reason.cause, readerFailure);
379
+ } else if (bodyKind === "stalled") {
380
+ assert.isTrue(Cause.isTimeoutError(underlying));
381
+ }
382
+ assert.isTrue(yield* Fiber.join(reader));
383
+ }).pipe(provideRefreshManager(client));
384
+ }),
385
+);
386
+
387
it.effect("a stalled token refresh fails and releases the credential lock", () =>
388
Effect.gen(function* () {
389
const manager = yield* CliTokenManager.CloudCliTokenManager;
@@ -192,7 +392,9 @@ it.effect("a stalled token refresh fails and releases the credential lock", () =
392
393
yield* TestClock.adjust(Duration.seconds(10));
394
195
- assert.instanceOf(yield* Fiber.join(refresh), CliTokenManager.CloudCliCredentialRefreshError);
395
+ const error = yield* Fiber.join(refresh);
396
+ assert.instanceOf(error, CliTokenManager.CloudCliCredentialRefreshError);
397
+ assert.isFalse(error.requiresLogin);
398
assert.isTrue(yield* Fiber.join(hasCredential));
399
}).pipe(
400
Effect.provide(
apps/server/src/cloud/CliTokenManager.ts
+88
-4
@@ -146,6 +146,21 @@ const OAuthErrorResponse = Schema.Struct({
146
error: Schema.String,
147
error_description: Schema.optional(Schema.String),
148
});
149
+const OAuthRefreshErrorCode = Schema.Literals([
150
+ "invalid_grant",
151
+ "invalid_client",
152
+ "unauthorized_client",
153
+ "unsupported_grant_type",
154
+ "invalid_request",
155
+ "invalid_scope",
156
+ "access_denied",
157
+]);
158
+const decodeOAuthRefreshErrorResponse = Schema.decodeUnknownOption(
159
+ Schema.Struct({ error: OAuthRefreshErrorCode }),
160
+);
161
+
162
+const isPermanentRefreshStatus = (status: number): boolean =>
163
+ status >= 400 && status < 500 && status !== 408 && status !== 429;
164
165
const DeviceAuthorizationResponse = Schema.Struct({
166
device_code: Schema.String,
@@ -190,13 +205,27 @@ export class CloudCliCredentialRemovalError extends Schema.TaggedError<CloudCliC
205
206
export class CloudCliCredentialRefreshError extends Schema.TaggedError<CloudCliCredentialRefreshError>()(
207
"CloudCliCredentialRefreshError",
193
- { cause: Schema.Defect() },
208
+ {
209
+ cause: Schema.Defect(),
210
+ status: Schema.optional(Schema.Number),
211
+ oauthErrorCode: Schema.optional(OAuthRefreshErrorCode),
212
+ },
213
) {
214
+ get requiresLogin(): boolean {
215
+ return this.status !== undefined && isPermanentRefreshStatus(this.status);
216
+ }
217
+
218
override get message(): string {
196
- return "Could not refresh the T3 Connect CLI credential.";
219
+ return this.requiresLogin
220
+ ? `Could not refresh the T3 Connect CLI credential${
221
+ this.oauthErrorCode === undefined ? "" : ` (OAuth error: ${this.oauthErrorCode})`
222
+ }. Run \`t3 connect login\` to authorize again.`
223
+ : "Could not refresh the T3 Connect CLI credential.";
224
}
225
}
226
227
+export const isCloudCliCredentialRefreshError = Schema.is(CloudCliCredentialRefreshError);
228
+
229
export class CloudCliCredentialReadError extends Schema.TaggedError<CloudCliCredentialReadError>()(
230
"CloudCliCredentialReadError",
231
{ cause: Schema.Defect() },
@@ -289,11 +318,52 @@ const exchangeToken = Effect.fn("cloud.cli_token.exchange")(function* (
318
params: Record<string, string>,
319
) {
320
const httpClient = (yield* HttpClient.HttpClient).pipe(HttpClient.filterStatusOk);
321
+ const startedAt = yield* Clock.monotonicTimeNanos;
322
return yield* HttpClientRequest.post(metadata.tokenEndpoint).pipe(
323
HttpClientRequest.bodyUrlParams(params),
324
httpClient.execute,
325
Effect.flatMap((response) => readTokenResponse(response, params)),
326
Effect.timeout(OAUTH_REQUEST_TIMEOUT),
327
+ Effect.catchTags({
328
+ HttpClientError: (cause) =>
329
+ Effect.gen(function* () {
330
+ const reason = cause.reason;
331
+ if (
332
+ params.grant_type !== "refresh_token" ||
333
+ reason._tag !== "StatusCodeError" ||
334
+ !isPermanentRefreshStatus(reason.response.status)
335
+ ) {
336
+ return yield* cause;
337
+ }
338
+ // Reading a rejection body shares the exchange budget without losing
339
+ // the known status when the body stalls or cannot be decoded.
340
+ const elapsed = Duration.toMillis(
341
+ Duration.nanos((yield* Clock.monotonicTimeNanos) - startedAt),
342
+ );
343
+ const body = yield* reason.response.json.pipe(
344
+ Effect.timeout(Math.max(0, Duration.toMillis(OAUTH_REQUEST_TIMEOUT) - elapsed)),
345
+ Effect.mapError(
346
+ (cause) =>
347
+ new CloudCliCredentialRefreshError({
348
+ status: reason.response.status,
349
+ cause: new HttpClientError.HttpClientError({
350
+ reason: new HttpClientError.StatusCodeError({
351
+ request: reason.request,
352
+ response: reason.response,
353
+ cause,
354
+ }),
355
+ }),
356
+ }),
357
+ ),
358
+ );
359
+ const diagnostic = decodeOAuthRefreshErrorResponse(body);
360
+ return yield* new CloudCliCredentialRefreshError({
361
+ cause,
362
+ status: reason.response.status,
363
+ ...(Option.isSome(diagnostic) ? { oauthErrorCode: diagnostic.value.error } : {}),
364
+ });
365
+ }),
366
+ }),
367
);
368
});
369
@@ -516,12 +586,26 @@ export const make = Effect.gen(function* () {
586
if (token.value.expiresAtEpochMs - CLOUD_CLI_OAUTH_REFRESH_EARLY_MS > now) {
587
return token;
588
}
519
- return Option.some(yield* refresh(token.value).pipe(Effect.flatMap(persist)));
589
+ return Option.some(
590
+ // The endpoint can rotate the refresh token before shutdown interrupts us.
591
+ // Keep the lock through persistence; the exchange still times out internally.
592
+ yield* refresh(token.value).pipe(Effect.flatMap(persist), Effect.uninterruptible),
593
+ );
594
});
595
596
const getExisting = semaphore.withPermits(1)(
597
getExistingNoLock().pipe(
524
- Effect.mapError((cause) => new CloudCliCredentialRefreshError({ cause })),
598
+ Effect.mapError((cause) =>
599
+ isCloudCliCredentialRefreshError(cause)
600
+ ? cause
601
+ : new CloudCliCredentialRefreshError({
602
+ cause,
603
+ ...(HttpClientError.isHttpClientError(cause) &&
604
+ cause.reason._tag === "StatusCodeError"
605
+ ? { status: cause.reason.response.status }
606
+ : {}),
607
+ }),
608
+ ),
609
Effect.provide(services),
610
),
611
);
apps/server/src/cloud/CloudLink.lifecycle.test.ts
+41
-1
@@ -242,6 +242,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
242
readonly respond?: () => Response;
243
readonly address?: HttpServer.HttpServer["Service"]["address"];
244
readonly respondEffect?: Effect.Effect<Response>;
245
+ readonly getExisting?: CliTokenManager.CloudCliTokenManager["Service"]["getExisting"];
246
}
247
248
const writeLauncherState = (update: ServiceUpdateRecord) =>
@@ -318,7 +319,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
319
CliTokenManager.CloudCliTokenManager,
320
CliTokenManager.CloudCliTokenManager.of({
321
get: unusedSecretStoreOperation(),
321
- getExisting: Effect.succeedSome(cliToken),
322
+ getExisting: harness.getExisting ?? Effect.succeedSome(cliToken),
323
hasCredential: unusedSecretStoreOperation(),
324
store: () => unusedSecretStoreOperation(),
325
clear: unusedSecretStoreOperation(),
@@ -1337,6 +1338,45 @@ describe("releaseManagedTunnelOnShutdown", () => {
1338
);
1339
});
1340
1341
+ it.effect("startup retries transient refresh failures but stops when login is required", () => {
1342
+ const { store } = makeMemorySecretStore(managedLinkSecrets);
1343
+ const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1344
+ const applyConfigCalls: Array<unknown> = [];
1345
+ const permanent = new CliTokenManager.CloudCliCredentialRefreshError({
1346
+ cause: new Error("private-oauth-failure"),
1347
+ status: 400,
1348
+ oauthErrorCode: "invalid_grant",
1349
+ });
1350
+ let attempts = 0;
1351
+ const getExisting = Effect.suspend(() => {
1352
+ attempts++;
1353
+ if (attempts > 2) return Effect.die("retried a permanent credential rejection");
1354
+ return Effect.fail(
1355
+ attempts === 2
1356
+ ? permanent
1357
+ : new CliTokenManager.CloudCliCredentialRefreshError({
1358
+ cause: new Error("connection reset"),
1359
+ }),
1360
+ );
1361
+ });
1362
+
1363
+ return Effect.gen(function* () {
1364
+ const link = yield* CloudLink.CloudLink;
1365
+ const registration = yield* retryManagedTunnelRegistration(
1366
+ link.reconcileDesiredLinkIfStillDesired("http://127.0.0.1:3773"),
1367
+ CloudLink.shouldRetryCloudLink,
1368
+ ).pipe(Effect.flip, Effect.forkChild);
1369
+ yield* TestClock.adjust("10 seconds");
1370
+ const error = yield* Fiber.join(registration);
1371
+ expect(attempts).toBe(2);
1372
+ expect(error).toBe(permanent);
1373
+ expect(error.message).toContain("t3 connect login");
1374
+ expect(error.message).not.toContain("private");
1375
+ expect(requests).toEqual([]);
1376
+ expect(applyConfigCalls).toEqual([]);
1377
+ }).pipe(provideReleaseHarness({ store, requests, applyConfigCalls, getExisting }));
1378
+ });
1379
+
1380
it.effect("keeps a tunnel configuration replaced during recovery", () => {
1381
const { store, values } = makeMemorySecretStore([
1382
[CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"],
apps/server/src/cloud/CloudLink.ts
+3
-1
@@ -291,7 +291,9 @@ const isPermanentLinkError = Schema.is(
291
);
292
293
export const shouldRetryCloudLink = (error: unknown): boolean =>
294
- shouldRetryRelayRequest(error) && !isPermanentLinkError(error);
294
+ shouldRetryRelayRequest(error) &&
295
+ !isPermanentLinkError(error) &&
296
+ !(CliTokenManager.isCloudCliCredentialRefreshError(error) && error.requiresLogin);
297
298
const rollbackFailed = (cause: unknown) =>
299
Effect.logWarning("Could not roll back a T3 Connect preference", { cause });