@devries-cameron20 / t3code Mirror · read-only / commits / 1aa46d0cdf
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 10:45 UTC

fix(connect): T3 Connect sockets no longer drop every hour (#27998)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Theo Browne committed Oct 31, 2026 at 01:04 UTC 1aa46d0cdfa4927a77b854752c62ed51868d3048
9 files changed +269 -27
apps/server/src/auth/SessionStore.test.ts
+99 -3
@@ -98,10 +98,10 @@ it.layer(NodeServices.layer)("SessionStore.layer", (it) => {
98 Effect.forkScoped({ startImmediately: true }),
99 );
100 const first = yield* sessions
101 - .awaitInvalidation(client.sessionId)
101 + .awaitInvalidation(client.sessionId, relaySessionInput.proofKeyThumbprint)
102 .pipe(Effect.forkScoped({ startImmediately: true }));
103 const second = yield* sessions
104 - .awaitInvalidation(client.sessionId)
104 + .awaitInvalidation(client.sessionId, relaySessionInput.proofKeyThumbprint)
105 .pipe(Effect.forkScoped({ startImmediately: true }));
106 if (operation === "revoke") yield* sessions.revoke(client.sessionId);
107 else if (operation === "revokeAllExcept") yield* sessions.revokeAllExcept(owner.sessionId);
@@ -120,7 +120,7 @@ it.layer(NodeServices.layer)("SessionStore.layer", (it) => {
120 yield* sessions.markConnected(client.sessionId);
121 const invalidated = yield* Deferred.make<void>();
122 const watcher = yield* sessions
123 - .awaitInvalidation(client.sessionId)
123 + .awaitInvalidation(client.sessionId, relaySessionInput.proofKeyThumbprint)
124 .pipe(
125 Effect.andThen(Deferred.succeed(invalidated, undefined)),
126 Effect.forkScoped({ startImmediately: true }),
@@ -133,6 +133,102 @@ it.layer(NodeServices.layer)("SessionStore.layer", (it) => {
133 }).pipe(Effect.scoped, Effect.provide(Layer.merge(layerSessionStore(), TestClock.layer()))),
134 );
135
136 + it.effect.each([
137 + { name: "a renewal for the same key", renewal: {}, closesAtMinute: 119 },
138 + {
139 + name: "a renewal for another key",
140 + renewal: { proofKeyThumbprint: "other-proof-key" },
141 + closesAtMinute: 60,
142 + },
143 + {
144 + name: "a renewal with fewer scopes",
145 + renewal: { scopes: ["orchestration:read"] as const },
146 + closesAtMinute: 60,
147 + },
148 + {
149 + name: "a renewal, until the original is revoked",
150 + renewal: {},
151 + closesAtMinute: 90,
152 + revoke: "original",
153 + },
154 + {
155 + name: "a renewal, until the renewal is revoked",
156 + renewal: {},
157 + closesAtMinute: 90,
158 + revoke: "renewal",
159 + },
160 + ] as const)("closes a DPoP socket at minute $closesAtMinute after $name", (testCase) =>
161 + Effect.gen(function* () {
162 + const sessions = yield* SessionStore.SessionStore;
163 + const client = yield* sessions.issue(relaySessionInput);
164 + const invalidated = yield* Deferred.make<void>();
165 + const watcher = yield* sessions
166 + .awaitInvalidation(client.sessionId, relaySessionInput.proofKeyThumbprint)
167 + .pipe(
168 + Effect.andThen(Deferred.succeed(invalidated, undefined)),
169 + Effect.forkScoped({ startImmediately: true }),
170 + );
171 + yield* TestClock.adjust(Duration.minutes(59));
172 + const renewal = yield* sessions.issue({ ...relaySessionInput, ...testCase.renewal });
173 + yield* TestClock.adjust(Duration.millis((testCase.closesAtMinute - 59) * 60_000 - 1));
174 + expect(yield* Deferred.isDone(invalidated)).toBe(false);
175 + yield* TestClock.adjust(Duration.millis(1));
176 + if ("revoke" in testCase) {
177 + yield* sessions.revoke(
178 + testCase.revoke === "original" ? client.sessionId : renewal.sessionId,
179 + );
180 + }
181 + yield* Fiber.join(watcher);
182 + }).pipe(Effect.scoped, Effect.provide(Layer.merge(layerSessionStore(), TestClock.layer()))),
183 + );
184 +
185 + it.effect("closes a DPoP socket when an earlier renewal in its chain is revoked", () =>
186 + Effect.gen(function* () {
187 + const sessions = yield* SessionStore.SessionStore;
188 + const client = yield* sessions.issue(relaySessionInput);
189 + const invalidated = yield* Deferred.make<void>();
190 + const watcher = yield* sessions
191 + .awaitInvalidation(client.sessionId, relaySessionInput.proofKeyThumbprint)
192 + .pipe(
193 + Effect.andThen(Deferred.succeed(invalidated, undefined)),
194 + Effect.forkScoped({ startImmediately: true }),
195 + );
196 + yield* TestClock.adjust(Duration.minutes(59));
197 + const first = yield* sessions.issue(relaySessionInput);
198 + yield* TestClock.adjust(Duration.minutes(59));
199 + yield* sessions.issue(relaySessionInput);
200 + // Past the first renewal's expiry, so the socket now follows the second.
201 + yield* TestClock.adjust(Duration.minutes(2));
202 + expect(yield* Deferred.isDone(invalidated)).toBe(false);
203 + yield* sessions.revoke(first.sessionId);
204 + yield* Fiber.join(watcher);
205 + }).pipe(Effect.scoped, Effect.provide(Layer.merge(layerSessionStore(), TestClock.layer()))),
206 + );
207 +
208 + it.effect("keeps a renewed DPoP socket when another socket on its expired session closes", () =>
209 + Effect.gen(function* () {
210 + const sessions = yield* SessionStore.SessionStore;
211 + const client = yield* sessions.issue(relaySessionInput);
212 + yield* sessions.markConnected(client.sessionId);
213 + const invalidated = yield* Deferred.make<void>();
214 + const watcher = yield* sessions
215 + .awaitInvalidation(client.sessionId, relaySessionInput.proofKeyThumbprint)
216 + .pipe(
217 + Effect.andThen(Deferred.succeed(invalidated, undefined)),
218 + Effect.forkScoped({ startImmediately: true }),
219 + );
220 + yield* TestClock.adjust(Duration.minutes(59));
221 + yield* sessions.issue(relaySessionInput);
222 + yield* TestClock.adjust(Duration.minutes(2));
223 + // An expired session that loses its last socket is reported as removed, not revoked.
224 + yield* sessions.markDisconnected(client.sessionId);
225 + yield* TestClock.adjust(Duration.millis(58 * 60_000 - 1));
226 + expect(yield* Deferred.isDone(invalidated)).toBe(false);
227 + yield* TestClock.adjust(Duration.millis(1));
228 + yield* Fiber.join(watcher);
229 + }).pipe(Effect.scoped, Effect.provide(Layer.merge(layerSessionStore(), TestClock.layer()))),
230 + );
231 +
232 it.effect("keys remote cookies by environment identity instead of state directory", () =>
233 Effect.gen(function* () {
234 const cookieName = (stateDir: string, environmentId: EnvironmentId) =>
apps/server/src/auth/SessionStore.ts
+57 -21
@@ -409,8 +409,10 @@ export class SessionStore extends Context.Service<
409 SessionCredentialInternalError
410 >;
411 readonly streamChanges: Stream.Stream<SessionCredentialChange>;
412 + /** Completes when a live socket's session is revoked or expires without a renewal for its DPoP key. */
413 readonly awaitInvalidation: (
414 sessionId: AuthSessionId,
415 + proofKeyThumbprint?: string,
416 ) => Effect.Effect<void, SessionCredentialVerificationError>;
417 readonly revoke: (
418 sessionId: AuthSessionId,
@@ -497,6 +499,9 @@ const make = Effect.gen(function* () {
499 const signingSecret = yield* secretStore.getOrCreateRandom(SIGNING_SECRET_NAME, 32);
500 const connectedSessionsRef = yield* Ref.make(new Map<AuthSessionId, number>());
501 const changesPubSub = yield* PubSub.unbounded<SessionCredentialChange>();
502 + // Newest session per DPoP key. A live socket moves to it at expiry, so a
503 + // client that renews its short-lived token keeps its socket.
504 + const latestSessionByProofKey = new Map<string, AuthSessionId>();
505 const cookieInput = {
506 mode: serverConfig.mode,
507 port: serverConfig.port,
@@ -703,6 +708,7 @@ const make = Effect.gen(function* () {
708 })
709 : authSessions.create(sessionRecord).pipe(Effect.as([] as ReadonlyArray<AuthSessionId>))
710 ).pipe(Effect.mapError((cause) => new SessionCredentialIssueError({ sessionId, cause })));
711 + if (claims.jkt) latestSessionByProofKey.set(claims.jkt, sessionId);
712 if (replacedSessionIds.length > 0) {
713 yield* Ref.update(connectedSessionsRef, (current) => {
714 const next = new Map(current);
@@ -1034,31 +1040,61 @@ const make = Effect.gen(function* () {
1040 return revokedSessionIds.length;
1041 });
1042
1037 - const awaitInvalidation = Effect.fn(function* (sessionId: AuthSessionId) {
1043 + const awaitInvalidation = Effect.fn(function* (
1044 + sessionId: AuthSessionId,
1045 + proofKeyThumbprint?: string,
1046 + ) {
1047 return yield* Effect.scoped(
1048 Effect.gen(function* () {
1049 // Subscribe before reading: revocation can race with the WebSocket upgrade.
1050 const subscription = yield* PubSub.subscribe(changesPubSub);
1042 - const row = yield* authSessions
1043 - .getById({ sessionId })
1044 - .pipe(
1045 - Effect.mapError(
1046 - (cause) => new SessionCredentialVerificationError({ sessionId, cause }),
1047 - ),
1048 - );
1049 - if (Option.isNone(row) || row.value.revokedAt !== null) return;
1050 - const now = yield* DateTime.now;
1051 - const remaining = row.value.expiresAt.epochMilliseconds - now.epochMilliseconds;
1052 - if (remaining <= 0) return;
1053 - yield* Effect.raceFirst(
1054 - Stream.fromSubscription(subscription).pipe(
1055 - Stream.filter(
1056 - (change) => change.type === "clientRemoved" && change.sessionId === sessionId,
1057 - ),
1058 - Stream.runHead,
1059 - ),
1060 - Effect.sleep(Duration.millis(remaining)),
1061 - );
1051 + const readRow = (id: AuthSessionId) =>
1052 + authSessions
1053 + .getById({ sessionId: id })
1054 + .pipe(
1055 + Effect.mapError(
1056 + (cause) => new SessionCredentialVerificationError({ sessionId: id, cause }),
1057 + ),
1058 + );
1059 + const original = yield* readRow(sessionId);
1060 + if (Option.isNone(original)) return;
1061 + // Revoking any session the socket has used closes it. Events only wake this loop
1062 + // and the rows decide: revocations are written before they are published, and
1063 + // clientRemoved also reports an expired session losing its last socket.
1064 + const chain = [sessionId];
1065 + const nextRemoval = Effect.gen(function* () {
1066 + while (true) {
1067 + const change = yield* PubSub.take(subscription);
1068 + if (change.type === "clientRemoved" && chain.includes(change.sessionId)) return;
1069 + }
1070 + });
1071 + let current = original.value;
1072 + while (true) {
1073 + const rows = yield* Effect.forEach(chain, readRow);
1074 + if (rows.some((row) => Option.isNone(row) || row.value.revokedAt !== null)) return;
1075 + const now = yield* DateTime.now;
1076 + const remaining = current.expiresAt.epochMilliseconds - now.epochMilliseconds;
1077 + if (remaining > 0) {
1078 + yield* Effect.raceFirst(nextRemoval, Effect.sleep(Duration.millis(remaining)));
1079 + continue;
1080 + }
1081 + const successorId =
1082 + proofKeyThumbprint === undefined
1083 + ? undefined
1084 + : latestSessionByProofKey.get(proofKeyThumbprint);
1085 + if (successorId === undefined || successorId === current.sessionId) return;
1086 + const successor = yield* readRow(successorId);
1087 + // The socket keeps its original scopes, so the renewal must still grant all of them.
1088 + if (
1089 + Option.isNone(successor) ||
1090 + successor.value.subject !== original.value.subject ||
1091 + !original.value.scopes.every((scope) => successor.value.scopes.includes(scope))
1092 + ) {
1093 + return;
1094 + }
1095 + chain.push(successorId);
1096 + current = successor.value;
1097 + }
1098 }),
1099 );
1100 });
apps/server/src/ws.ts
+1 -1
@@ -3385,7 +3385,7 @@ export const layer = Layer.unwrap(
3385 () =>
3386 Effect.raceFirst(
3387 rpcWebSocketHttpEffect,
3388 - sessions.awaitInvalidation(session.sessionId).pipe(
3388 + sessions.awaitInvalidation(session.sessionId, session.proofKeyThumbprint).pipe(
3389 Effect.as(HttpServerResponse.empty()),
3390 Effect.catchTags({
3391 SessionCredentialVerificationError: (error) =>
docs/internals/connection-runtime.md
+9 -2
@@ -46,8 +46,15 @@ RPC sessions authenticate at socket upgrade, while HTTP requests need current
46 credentials from the
47 [authorization service](../../packages/client-runtime/src/authorization/service.ts).
48 Replacing a healthy socket for HTTP renewal would interrupt conversations and
49 -change the transport generation without a transport failure. Credential expiry
50 -does not close the socket, and refresh failure belongs to the HTTP operation.
49 +change the transport generation without a transport failure. Refresh failure
50 +belongs to the HTTP operation.
51 +
52 +The server closes a socket when its session is revoked or expires. T3 Connect
53 +access tokens last one hour, so the driver renews a live socket's token in its
54 +last minute. The server then moves the socket to the newer session for the same
55 +DPoP key, subject, and scopes
56 +([`SessionStore.awaitInvalidation`](../../apps/server/src/auth/SessionStore.ts)).
57 +If renewal fails, the socket closes at expiry and the supervisor reconnects.
58
59 Session listings must retain unrevoked connected sessions after credential expiry
60 so an open connection does not disappear from connection management. This does
packages/client-runtime/src/authorization/layer.test.ts
+53
@@ -602,6 +602,59 @@ describe("RemoteEnvironmentAuthorization", () => {
602 }),
603 );
604
605 + it.effect("renews a live socket's token in the minute before each expiry", () =>
606 + Effect.gen(function* () {
607 + const puts = yield* Queue.unbounded<void>();
608 + const relayBlips = yield* Ref.make(1);
609 + const harness = yield* makeHarness({
610 + beforeBootstrap: Ref.getAndUpdate(relayBlips, (count) => count - 1).pipe(
611 + Effect.flatMap((count) =>
612 + count > 0
613 + ? Effect.fail(
614 + new ManagedRelay.ManagedRelayRequestTimeoutError({
615 + activity: "Relay environment connection",
616 + timeoutMs: 30_000,
617 + traceId: null,
618 + }),
619 + )
620 + : Effect.void,
621 + ),
622 + ),
623 + initialToken: persistedToken({ expiresAtEpochMs: 3_600_000 }),
624 + responses: [
625 + Response.json(DESCRIPTOR),
626 + accessToken("renewed-1"),
627 + Response.json(DESCRIPTOR),
628 + accessToken("renewed-2"),
629 + ],
630 + beforePut: Queue.offer(puts, undefined).pipe(Effect.asVoid),
631 + });
632 +
633 + yield* Effect.gen(function* () {
634 + const remote = yield* RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization;
635 + yield* remote
636 + .renewDpopBeforeExpiry({
637 + expectedEnvironmentId: ENVIRONMENT_ID,
638 + expiresAtEpochMs: 3_600_000,
639 + })
640 + .pipe(Effect.forkScoped);
641 + yield* TestClock.adjust(3_540_000 - 1);
642 + expect(yield* Ref.get(harness.bootstrapCalls)).toBe(0);
643 + // The first try hits a relay blip, and the retry 5 s later succeeds.
644 + yield* TestClock.adjust(1);
645 + expect(yield* Ref.get(harness.bootstrapCalls)).toBe(1);
646 + yield* TestClock.adjust(5_000);
647 + yield* Queue.take(puts);
648 + // Renewed at 0:59:05, so the next renewal is at 1:58:05.
649 + yield* TestClock.adjust(7_085_000 - 3_545_000 - 1);
650 + expect(yield* Ref.get(harness.bootstrapCalls)).toBe(2);
651 + yield* TestClock.adjust(1);
652 + yield* Queue.take(puts);
653 + expect(yield* Ref.get(harness.bootstrapCalls)).toBe(3);
654 + }).pipe(Effect.scoped, Effect.provide(Layer.merge(harness.layer, TestClock.layer())));
655 + }),
656 + );
657 +
658 it.effect("evicts an auth-invalid cached token and obtains a fresh bootstrap", () =>
659 Effect.gen(function* () {
660 const cached = new TokenStore.RemoteDpopAccessToken({
packages/client-runtime/src/authorization/service.ts
+32
@@ -82,6 +82,14 @@ export class RemoteEnvironmentAuthorization extends Context.Service<
82 readonly expectedEnvironmentId: EnvironmentId;
83 readonly rejectedAccessToken?: string;
84 }) => Effect.Effect<AuthorizedRemoteHttpEnvironment, ConnectionAttemptError>;
85 + /**
86 + * Renews a live socket's token before it expires, so the server moves the
87 + * socket to the new session instead of closing it. Runs until interrupted.
88 + */
89 + readonly renewDpopBeforeExpiry: (input: {
90 + readonly expectedEnvironmentId: EnvironmentId;
91 + readonly expiresAtEpochMs: number;
92 + }) => Effect.Effect<void, ConnectionBlockedError>;
93 }
94 >()("@t3tools/client-runtime/authorization/service/RemoteEnvironmentAuthorization") {}
95
@@ -530,6 +538,30 @@ export const make = Effect.gen(function* () {
538 withRelayClientTracing,
539 );
540 }),
541 + // Untraced: a span here would stay open for the socket's lifetime. Each renewal has its own.
542 + renewDpopBeforeExpiry: Effect.fnUntraced(function* (input) {
543 + let expiresAtEpochMs = input.expiresAtEpochMs;
544 + let now = yield* Clock.currentTimeMillis;
545 + // Past expiry the server has closed the socket.
546 + while (now < expiresAtEpochMs) {
547 + // getDpopToken reuses a cached token until it is within the skew of expiry.
548 + // The floor spaces out retries and keeps a wall-clock jump from spinning.
549 + yield* Effect.sleep(
550 + Math.max(5_000, expiresAtEpochMs - DPOP_ACCESS_TOKEN_REFRESH_SKEW_MS - now),
551 + );
552 + const renewed = yield* getDpopToken({
553 + expectedEnvironmentId: input.expectedEnvironmentId,
554 + }).pipe(
555 + Effect.map(({ token }) => token.expiresAtEpochMs),
556 + Effect.withSpan("environment.authorization.renewSocket"),
557 + withRelayClientTracing,
558 + // A relay blip retries until expiry; a blocked credential ends the loop.
559 + Effect.catchTags({ ConnectionTransientError: () => Effect.succeed(expiresAtEpochMs) }),
560 + );
561 + expiresAtEpochMs = Math.max(expiresAtEpochMs, renewed);
562 + now = yield* Clock.currentTimeMillis;
563 + }
564 + }),
565 });
566 });
567
packages/client-runtime/src/connection/driver.ts
+16
@@ -22,6 +22,7 @@ import {
22 sendsTokenInClear,
23 usableRoutes,
24 } from "./routes.ts";
25 +import * as RemoteEnvironmentAuthorization from "../authorization/service.ts";
26 import * as RpcSession from "../rpc/session.ts";
27 import { fetchRemoteEnvironmentDescriptor } from "../environment/descriptor.ts";
28
@@ -141,6 +142,7 @@ export const connectOverRoutes = Effect.fn("ConnectionDriver.connectOverRoutes")
142 export const make = Effect.gen(function* () {
143 const resolver = yield* ConnectionResolver.ConnectionResolver;
144 const sessions = yield* RpcSession.RpcSessionFactory;
145 + const remote = yield* RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization;
146 const httpClient = yield* HttpClient.HttpClient;
147
148 // The descriptor is public, so this sends no credential to whatever
@@ -196,6 +198,20 @@ export const make = Effect.gen(function* () {
198 const session = yield* sessions.connect(prepared);
199 yield* reportProgress({ stage: "synchronizing", prepared });
200 yield* session.ready;
201 + // The server closes a socket when its session expires, unless the same key renewed it.
202 + if (prepared.httpAuthorization?._tag === "Dpop") {
203 + yield* remote
204 + .renewDpopBeforeExpiry({
205 + expectedEnvironmentId: prepared.environmentId,
206 + expiresAtEpochMs: prepared.httpAuthorization.expiresAtEpochMs,
207 + })
208 + .pipe(
209 + Effect.catch((error) =>
210 + Effect.logWarning("Could not renew the connection's credential.", { error }),
211 + ),
212 + Effect.forkScoped,
213 + );
214 + }
215 return { prepared, session } satisfies EnvironmentConnectionLease;
216 }),
217 );
packages/client-runtime/src/connection/resolver.test.ts
+1
@@ -108,6 +108,7 @@ const makeDependencies = Effect.fn("TestConnectionResolver.makeDependencies")((o
108 },
109 })),
110 authorizeDpopHttp: () => Effect.die("unused"),
111 + renewDpopBeforeExpiry: () => Effect.die("unused"),
112 });
113 const ssh = ClientCapabilities.SshEnvironmentGateway.of({
114 provision: () => Effect.die("unused"),
packages/client-runtime/src/state/environmentHttpAuth.test.ts
+1
@@ -146,6 +146,7 @@ function makeHarness(reply: (requestNumber: number) => Response | Promise<Respon
146 },
147 };
148 }),
149 + renewDpopBeforeExpiry: () => Effect.die("HTTP requests must not renew a socket's token."),
150 });
151 const signer = ManagedRelay.ManagedRelayDpopSigner.of({
152 thumbprint: Effect.succeed("test-thumbprint"),