The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 13:15 UTC
fix(relay): concurrent link requests can no longer exceed the managed tunnel limit (#26186)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Oct 27, 2026 at 23:14 UTC
9bc0419cc026be92c6273e402d2ec2a3e70f2efe
5 files changed
+284
-105
infra/relay/src/environments/ManagedEndpointAllocations.ts
+9
-21
@@ -255,7 +255,14 @@ const make = Effect.gen(function* () {
255
createdAt: now,
256
updatedAt: now,
257
})
258
- .onConflictDoNothing()
258
+ // Refreshing updatedAt makes a re-reserved row count toward the tunnel limit again.
259
+ .onConflictDoUpdate({
260
+ target: [
261
+ relayManagedEndpointAllocations.userId,
262
+ relayManagedEndpointAllocations.environmentId,
263
+ ],
264
+ set: { updatedAt: now },
265
+ })
266
.returning(allocationSelection)
267
.pipe(
268
Effect.mapError(
@@ -269,26 +276,7 @@ const make = Effect.gen(function* () {
276
),
277
);
278
272
- const allocation =
273
- inserted[0] ??
274
- (yield* db
275
- .select(allocationSelection)
276
- .from(relayManagedEndpointAllocations)
277
- .where(whereAllocation(input))
278
- .limit(1)
279
- .pipe(
280
- Effect.map((rows) => rows[0]),
281
- Effect.mapError(
282
- (cause) =>
283
- new ManagedEndpointAllocationPersistenceError({
284
- operation: "reserve",
285
- stage: "database-request",
286
- ...input,
287
- cause,
288
- }),
289
- ),
290
- ));
291
-
279
+ const allocation = inserted[0];
280
if (allocation === undefined) {
281
return yield* new ManagedEndpointAllocationPersistenceError({
282
operation: "reserve",
infra/relay/src/environments/ManagedEndpointProvider.test.ts
+6
-3
@@ -335,10 +335,13 @@ function makeTunnelLimits(
335
result: ManagedTunnelLimits.ManagedTunnelLimitExceeded | null = null,
336
) {
337
return ManagedTunnelLimits.ManagedTunnelLimits.of({
338
- ensureCapacity: (input) =>
339
- Effect.suspend(() => {
338
+ withCapacity: (input, reserve) =>
339
+ Effect.gen(function* () {
340
calls.push(input);
341
- return result === null ? Effect.void : Effect.fail(result);
341
+ if (result !== null) {
342
+ return yield* result;
343
+ }
344
+ return yield* reserve;
345
}),
346
});
347
}
infra/relay/src/environments/ManagedEndpointProvider.ts
+21
-21
@@ -1169,27 +1169,7 @@ const make = Effect.gen(function* () {
1169
environmentHash,
1170
);
1171
const requestedTunnelName = managedEndpointTunnelName(cf.namespace, environmentHash);
1172
- yield* tunnelLimits
1173
- .ensureCapacity({
1174
- userId: input.userId,
1175
- environmentId: input.environmentId,
1176
- })
1177
- .pipe(
1178
- Effect.catchTags({
1179
- ManagedTunnelLimitPersistenceError: (cause) =>
1180
- Effect.fail(
1181
- new ManagedEndpointProvisioningFailed({
1182
- userId: input.userId,
1183
- environmentId: input.environmentId,
1184
- stage: "check-tunnel-limit",
1185
- hostname: requestedHostname,
1186
- tunnelName: requestedTunnelName,
1187
- cause,
1188
- }),
1189
- ),
1190
- }),
1191
- );
1192
- const allocation = yield* allocations
1172
+ const reserveAllocation = allocations
1173
.reserve({
1174
userId: input.userId,
1175
environmentId: input.environmentId,
@@ -1209,6 +1189,26 @@ const make = Effect.gen(function* () {
1189
}),
1190
),
1191
);
1192
+ const allocation = yield* tunnelLimits
1193
+ .withCapacity(
1194
+ { userId: input.userId, environmentId: input.environmentId },
1195
+ reserveAllocation,
1196
+ )
1197
+ .pipe(
1198
+ Effect.catchTags({
1199
+ ManagedTunnelLimitPersistenceError: (cause) =>
1200
+ Effect.fail(
1201
+ new ManagedEndpointProvisioningFailed({
1202
+ userId: input.userId,
1203
+ environmentId: input.environmentId,
1204
+ stage: "check-tunnel-limit",
1205
+ hostname: requestedHostname,
1206
+ tunnelName: requestedTunnelName,
1207
+ cause,
1208
+ }),
1209
+ ),
1210
+ }),
1211
+ );
1212
const { hostname, tunnelName } = allocation;
1213
1214
if (input.reuseReadyTunnel === true) {
infra/relay/src/environments/ManagedTunnelLimits.test.ts
new
+115
@@ -0,0 +1,115 @@
1
+import { describe, expect, it } from "@effect/vitest";
2
+import * as Context from "effect/Context";
3
+import * as Effect from "effect/Effect";
4
+import * as Layer from "effect/Layer";
5
+import * as Semaphore from "effect/Semaphore";
6
+import { PgDialect } from "drizzle-orm/pg-core";
7
+
8
+import * as RelayDb from "../db.ts";
9
+import { relayManagedEndpointAllocations } from "../persistence/schema.ts";
10
+import * as ManagedTunnelLimits from "./ManagedTunnelLimits.ts";
11
+
12
+class FakeTransaction extends Context.Service<FakeTransaction, { holdsLock: boolean }>()(
13
+ "t3code-relay/environments/ManagedTunnelLimits.test/FakeTransaction",
14
+) {}
15
+
16
+// Models Postgres just enough: the advisory lock only works inside a transaction and is held until it ends.
17
+function makeFakeDb(reserved: ReadonlyArray<unknown>, countConditions: Array<unknown> = []) {
18
+ const userLock = Semaphore.makeUnsafe(1);
19
+ return {
20
+ $client: {
21
+ withTransaction: <A, E, R>(effect: Effect.Effect<A, E, R>) => {
22
+ const transaction = { holdsLock: false };
23
+ return effect.pipe(
24
+ Effect.provideService(FakeTransaction, transaction),
25
+ Effect.ensuring(
26
+ Effect.suspend(() => (transaction.holdsLock ? userLock.release(1) : Effect.void)),
27
+ ),
28
+ );
29
+ },
30
+ },
31
+ execute: () =>
32
+ Effect.gen(function* () {
33
+ const transaction = yield* Effect.serviceOption(FakeTransaction);
34
+ if (transaction._tag === "None") {
35
+ return yield* Effect.die("advisory lock taken outside a transaction");
36
+ }
37
+ yield* userLock.take(1);
38
+ transaction.value.holdsLock = true;
39
+ return [];
40
+ }),
41
+ select: () => ({
42
+ from: (table: unknown) => ({
43
+ where: (condition: unknown) => {
44
+ if (table !== relayManagedEndpointAllocations) {
45
+ return { limit: () => Effect.succeed([]) };
46
+ }
47
+ countConditions.push(condition);
48
+ return Effect.yieldNow.pipe(Effect.as([{ activeTunnels: reserved.length }]));
49
+ },
50
+ }),
51
+ }),
52
+ } as unknown as RelayDb.RelayDb["Service"];
53
+}
54
+
55
+describe("ManagedTunnelLimits", () => {
56
+ it.effect("rejects concurrent reservations past the limit", () => {
57
+ const reserved: Array<string> = [];
58
+ const attempts = ManagedTunnelLimits.DEFAULT_MANAGED_TUNNEL_LIMIT + 2;
59
+
60
+ return Effect.gen(function* () {
61
+ const limits = yield* ManagedTunnelLimits.ManagedTunnelLimits;
62
+ const results = yield* Effect.forEach(
63
+ Array.from({ length: attempts }, (_, index) => `environment-${index}`),
64
+ (environmentId) =>
65
+ limits
66
+ .withCapacity(
67
+ { userId: "user-1", environmentId },
68
+ Effect.yieldNow.pipe(Effect.andThen(Effect.sync(() => reserved.push(environmentId)))),
69
+ )
70
+ .pipe(Effect.result),
71
+ { concurrency: "unbounded" },
72
+ );
73
+
74
+ expect(reserved).toHaveLength(ManagedTunnelLimits.DEFAULT_MANAGED_TUNNEL_LIMIT);
75
+ expect(
76
+ results.filter(
77
+ (result) =>
78
+ result._tag === "Failure" && result.failure._tag === "ManagedTunnelLimitExceeded",
79
+ ),
80
+ ).toHaveLength(2);
81
+ }).pipe(
82
+ Effect.provide(
83
+ ManagedTunnelLimits.layer.pipe(
84
+ Layer.provide(Layer.succeed(RelayDb.RelayDb, makeFakeDb(reserved))),
85
+ ),
86
+ ),
87
+ );
88
+ });
89
+
90
+ it.effect("pending reservations stop counting once released or withdrawn", () => {
91
+ const countConditions: Array<unknown> = [];
92
+
93
+ return Effect.gen(function* () {
94
+ const limits = yield* ManagedTunnelLimits.ManagedTunnelLimits;
95
+ yield* limits.withCapacity({ userId: "user-1", environmentId: "environment-1" }, Effect.void);
96
+
97
+ expect(countConditions).toHaveLength(1);
98
+ const query = new PgDialect().sqlToQuery(countConditions[0] as never);
99
+ // claimRelease writes updatedAt = tunnelReleasedAt; a later reserve moves updatedAt past it.
100
+ expect(query.sql).toContain(
101
+ '"relay_managed_endpoint_allocations"."updated_at" > "relay_managed_endpoint_allocations"."tunnel_released_at"',
102
+ );
103
+ // A link revoked or made publish-only after the last reservation no longer holds a slot.
104
+ expect(query.sql).toContain(
105
+ '"relay_environment_links"."updated_at" >= "relay_managed_endpoint_allocations"."updated_at"',
106
+ );
107
+ }).pipe(
108
+ Effect.provide(
109
+ ManagedTunnelLimits.layer.pipe(
110
+ Layer.provide(Layer.succeed(RelayDb.RelayDb, makeFakeDb([], countConditions))),
111
+ ),
112
+ ),
113
+ );
114
+ });
115
+});
infra/relay/src/environments/ManagedTunnelLimits.ts
+133
-60
@@ -1,8 +1,11 @@
1
-import { and, count, eq, isNull, ne } from "drizzle-orm";
1
+import { and, count, eq, exists, gt, gte, isNotNull, isNull, ne, not, or, sql } from "drizzle-orm";
2
+import { QueryBuilder } from "drizzle-orm/pg-core";
3
import * as Context from "effect/Context";
4
+import * as DateTime from "effect/DateTime";
5
import * as Effect from "effect/Effect";
6
import * as Layer from "effect/Layer";
7
import * as Schema from "effect/Schema";
8
+import * as SqlError from "effect/sql/SqlError";
9
10
import * as RelayDb from "../db.ts";
11
import {
@@ -11,12 +14,14 @@ import {
14
relayManagedTunnelLimits,
15
} from "../persistence/schema.ts";
16
14
-const DEFAULT_MANAGED_TUNNEL_LIMIT = 3;
17
+export const DEFAULT_MANAGED_TUNNEL_LIMIT = 3;
18
+// Reserve-to-tunnel takes two Cloudflare calls capped at 8 seconds each; a reservation older than this was abandoned.
19
+const MANAGED_TUNNEL_PENDING_WINDOW_MINUTES = 15;
20
21
export class ManagedTunnelLimitPersistenceError extends Schema.TaggedError<ManagedTunnelLimitPersistenceError>()(
22
"ManagedTunnelLimitPersistenceError",
23
{
19
- operation: Schema.Literals(["load-limit", "count-tunnels"]),
24
+ operation: Schema.Literals(["transaction", "lock-user", "load-limit", "count-tunnels"]),
25
userId: Schema.String,
26
cause: Schema.Defect(),
27
},
@@ -43,10 +48,11 @@ export class ManagedTunnelLimitExceeded extends Schema.TaggedError<ManagedTunnel
48
export class ManagedTunnelLimits extends Context.Service<
49
ManagedTunnelLimits,
50
{
46
- readonly ensureCapacity: (input: {
47
- readonly userId: string;
48
- readonly environmentId: string;
49
- }) => Effect.Effect<void, ManagedTunnelLimitExceeded | ManagedTunnelLimitPersistenceError>;
51
+ /** Runs `reserve` only while the user is under their limit, serialized per user in one transaction. */
52
+ readonly withCapacity: <A, E, R>(
53
+ input: { readonly userId: string; readonly environmentId: string },
54
+ reserve: Effect.Effect<A, E, R>,
55
+ ) => Effect.Effect<A, E | ManagedTunnelLimitExceeded | ManagedTunnelLimitPersistenceError, R>;
56
}
57
>()("t3code-relay/environments/ManagedTunnelLimits") {}
58
@@ -54,63 +60,130 @@ const make = Effect.gen(function* () {
60
const db = yield* RelayDb.RelayDb;
61
62
return ManagedTunnelLimits.of({
57
- ensureCapacity: Effect.fn("relay.managed_tunnel_limits.ensure_capacity")(function* (input) {
58
- const overrides = yield* db
59
- .select({ maxTunnels: relayManagedTunnelLimits.maxTunnels })
60
- .from(relayManagedTunnelLimits)
61
- .where(eq(relayManagedTunnelLimits.userId, input.userId))
62
- .limit(1)
63
- .pipe(
64
- Effect.mapError(
65
- (cause) =>
66
- new ManagedTunnelLimitPersistenceError({
67
- operation: "load-limit",
68
- userId: input.userId,
69
- cause,
70
- }),
71
- ),
63
+ withCapacity: Effect.fn("relay.managed_tunnel_limits.with_capacity")(function* <A, E, R>(
64
+ input: { readonly userId: string; readonly environmentId: string },
65
+ reserve: Effect.Effect<A, E, R>,
66
+ ): Effect.fn.Return<A, E | ManagedTunnelLimitExceeded | ManagedTunnelLimitPersistenceError, R> {
67
+ const checkAndReserve = Effect.gen(function* () {
68
+ // Held until commit, so the count below sees every reservation that took the lock first.
69
+ yield* db
70
+ .execute(
71
+ sql`select pg_advisory_xact_lock(hashtextextended(${`managed-tunnel-limit:${input.userId}`}, 0))`,
72
+ )
73
+ .pipe(
74
+ Effect.mapError(
75
+ (cause) =>
76
+ new ManagedTunnelLimitPersistenceError({
77
+ operation: "lock-user",
78
+ userId: input.userId,
79
+ cause,
80
+ }),
81
+ ),
82
+ );
83
+
84
+ const overrides = yield* db
85
+ .select({ maxTunnels: relayManagedTunnelLimits.maxTunnels })
86
+ .from(relayManagedTunnelLimits)
87
+ .where(eq(relayManagedTunnelLimits.userId, input.userId))
88
+ .limit(1)
89
+ .pipe(
90
+ Effect.mapError(
91
+ (cause) =>
92
+ new ManagedTunnelLimitPersistenceError({
93
+ operation: "load-limit",
94
+ userId: input.userId,
95
+ cause,
96
+ }),
97
+ ),
98
+ );
99
+ const maxTunnels = overrides[0]?.maxTunnels ?? DEFAULT_MANAGED_TUNNEL_LIMIT;
100
+
101
+ const pendingCutoff = DateTime.formatIso(
102
+ DateTime.subtract(yield* DateTime.now, {
103
+ minutes: MANAGED_TUNNEL_PENDING_WINDOW_MINUTES,
104
+ }),
105
);
73
- const maxTunnels = overrides[0]?.maxTunnels ?? DEFAULT_MANAGED_TUNNEL_LIMIT;
106
+ // Counts allocations, not links: a link is written only after its tunnel exists.
107
+ // Excludes this environment so relinking at the limit stays idempotent.
108
+ const allocation = relayManagedEndpointAllocations;
109
+ const withdrawnLink = (since?: typeof allocation.updatedAt) =>
110
+ exists(
111
+ new QueryBuilder()
112
+ .select({ userId: relayEnvironmentLinks.userId })
113
+ .from(relayEnvironmentLinks)
114
+ .where(
115
+ and(
116
+ eq(relayEnvironmentLinks.userId, allocation.userId),
117
+ eq(relayEnvironmentLinks.environmentId, allocation.environmentId),
118
+ or(
119
+ isNotNull(relayEnvironmentLinks.revokedAt),
120
+ eq(relayEnvironmentLinks.managedTunnelsEnabled, false),
121
+ ),
122
+ since === undefined ? undefined : gte(relayEnvironmentLinks.updatedAt, since),
123
+ ),
124
+ ),
125
+ );
126
+ const counted = yield* db
127
+ .select({ activeTunnels: count() })
128
+ .from(allocation)
129
+ .where(
130
+ and(
131
+ eq(allocation.userId, input.userId),
132
+ ne(allocation.environmentId, input.environmentId),
133
+ or(
134
+ // Reserved but not yet holding a tunnel. Release writes updatedAt = tunnelReleasedAt and
135
+ // a relink reserves before rewriting its link, so only a newer reservation counts here.
136
+ and(
137
+ gt(allocation.updatedAt, pendingCutoff),
138
+ or(
139
+ isNull(allocation.tunnelReleasedAt),
140
+ gt(allocation.updatedAt, allocation.tunnelReleasedAt),
141
+ ),
142
+ not(withdrawnLink(allocation.updatedAt)),
143
+ ),
144
+ and(
145
+ isNotNull(allocation.tunnelId),
146
+ isNull(allocation.tunnelReleasedAt),
147
+ not(withdrawnLink()),
148
+ ),
149
+ ),
150
+ ),
151
+ )
152
+ .pipe(
153
+ Effect.mapError(
154
+ (cause) =>
155
+ new ManagedTunnelLimitPersistenceError({
156
+ operation: "count-tunnels",
157
+ userId: input.userId,
158
+ cause,
159
+ }),
160
+ ),
161
+ );
162
+ const activeTunnels = counted[0]?.activeTunnels ?? 0;
163
75
- // Exclude this environment so relinking remains idempotent at the account limit.
76
- const counted = yield* db
77
- .select({ activeTunnels: count() })
78
- .from(relayManagedEndpointAllocations)
79
- .innerJoin(
80
- relayEnvironmentLinks,
81
- and(
82
- eq(relayEnvironmentLinks.userId, relayManagedEndpointAllocations.userId),
83
- eq(relayEnvironmentLinks.environmentId, relayManagedEndpointAllocations.environmentId),
84
- ),
85
- )
86
- .where(
87
- and(
88
- eq(relayManagedEndpointAllocations.userId, input.userId),
89
- ne(relayManagedEndpointAllocations.environmentId, input.environmentId),
90
- isNull(relayEnvironmentLinks.revokedAt),
91
- eq(relayEnvironmentLinks.managedTunnelsEnabled, true),
92
- ),
93
- )
94
- .pipe(
95
- Effect.mapError(
96
- (cause) =>
97
- new ManagedTunnelLimitPersistenceError({
98
- operation: "count-tunnels",
164
+ if (activeTunnels >= maxTunnels) {
165
+ return yield* new ManagedTunnelLimitExceeded({
166
+ userId: input.userId,
167
+ environmentId: input.environmentId,
168
+ maxTunnels,
169
+ activeTunnels,
170
+ });
171
+ }
172
+ return yield* reserve;
173
+ });
174
+
175
+ // Statements inside map their own SqlErrors, so any left here come from opening or committing the transaction.
176
+ return yield* db.$client.withTransaction(checkAndReserve).pipe(
177
+ Effect.mapError((cause) =>
178
+ SqlError.isSqlError(cause)
179
+ ? new ManagedTunnelLimitPersistenceError({
180
+ operation: "transaction",
181
userId: input.userId,
182
cause,
101
- }),
102
- ),
103
- );
104
- const activeTunnels = counted[0]?.activeTunnels ?? 0;
105
-
106
- if (activeTunnels >= maxTunnels) {
107
- return yield* new ManagedTunnelLimitExceeded({
108
- userId: input.userId,
109
- environmentId: input.environmentId,
110
- maxTunnels,
111
- activeTunnels,
112
- });
113
- }
183
+ })
184
+ : cause,
185
+ ),
186
+ );
187
}),
188
});
189
});