@devries-cameron20 / t3code Mirror · read-only / commits / 9bc0419cc0
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 13:15 UTC

fix(relay): concurrent link requests can no longer exceed the managed tunnel limit (#26186)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 27, 2026 at 23:14 UTC 9bc0419cc026be92c6273e402d2ec2a3e70f2efe
5 files changed +284 -105
infra/relay/src/environments/ManagedEndpointAllocations.ts
+9 -21
@@ -255,7 +255,14 @@ const make = Effect.gen(function* () {
255 createdAt: now,
256 updatedAt: now,
257 })
258 - .onConflictDoNothing()
258 + // Refreshing updatedAt makes a re-reserved row count toward the tunnel limit again.
259 + .onConflictDoUpdate({
260 + target: [
261 + relayManagedEndpointAllocations.userId,
262 + relayManagedEndpointAllocations.environmentId,
263 + ],
264 + set: { updatedAt: now },
265 + })
266 .returning(allocationSelection)
267 .pipe(
268 Effect.mapError(
@@ -269,26 +276,7 @@ const make = Effect.gen(function* () {
276 ),
277 );
278
272 - const allocation =
273 - inserted[0] ??
274 - (yield* db
275 - .select(allocationSelection)
276 - .from(relayManagedEndpointAllocations)
277 - .where(whereAllocation(input))
278 - .limit(1)
279 - .pipe(
280 - Effect.map((rows) => rows[0]),
281 - Effect.mapError(
282 - (cause) =>
283 - new ManagedEndpointAllocationPersistenceError({
284 - operation: "reserve",
285 - stage: "database-request",
286 - ...input,
287 - cause,
288 - }),
289 - ),
290 - ));
291 -
279 + const allocation = inserted[0];
280 if (allocation === undefined) {
281 return yield* new ManagedEndpointAllocationPersistenceError({
282 operation: "reserve",
infra/relay/src/environments/ManagedEndpointProvider.test.ts
+6 -3
@@ -335,10 +335,13 @@ function makeTunnelLimits(
335 result: ManagedTunnelLimits.ManagedTunnelLimitExceeded | null = null,
336 ) {
337 return ManagedTunnelLimits.ManagedTunnelLimits.of({
338 - ensureCapacity: (input) =>
339 - Effect.suspend(() => {
338 + withCapacity: (input, reserve) =>
339 + Effect.gen(function* () {
340 calls.push(input);
341 - return result === null ? Effect.void : Effect.fail(result);
341 + if (result !== null) {
342 + return yield* result;
343 + }
344 + return yield* reserve;
345 }),
346 });
347 }
infra/relay/src/environments/ManagedEndpointProvider.ts
+21 -21
@@ -1169,27 +1169,7 @@ const make = Effect.gen(function* () {
1169 environmentHash,
1170 );
1171 const requestedTunnelName = managedEndpointTunnelName(cf.namespace, environmentHash);
1172 - yield* tunnelLimits
1173 - .ensureCapacity({
1174 - userId: input.userId,
1175 - environmentId: input.environmentId,
1176 - })
1177 - .pipe(
1178 - Effect.catchTags({
1179 - ManagedTunnelLimitPersistenceError: (cause) =>
1180 - Effect.fail(
1181 - new ManagedEndpointProvisioningFailed({
1182 - userId: input.userId,
1183 - environmentId: input.environmentId,
1184 - stage: "check-tunnel-limit",
1185 - hostname: requestedHostname,
1186 - tunnelName: requestedTunnelName,
1187 - cause,
1188 - }),
1189 - ),
1190 - }),
1191 - );
1192 - const allocation = yield* allocations
1172 + const reserveAllocation = allocations
1173 .reserve({
1174 userId: input.userId,
1175 environmentId: input.environmentId,
@@ -1209,6 +1189,26 @@ const make = Effect.gen(function* () {
1189 }),
1190 ),
1191 );
1192 + const allocation = yield* tunnelLimits
1193 + .withCapacity(
1194 + { userId: input.userId, environmentId: input.environmentId },
1195 + reserveAllocation,
1196 + )
1197 + .pipe(
1198 + Effect.catchTags({
1199 + ManagedTunnelLimitPersistenceError: (cause) =>
1200 + Effect.fail(
1201 + new ManagedEndpointProvisioningFailed({
1202 + userId: input.userId,
1203 + environmentId: input.environmentId,
1204 + stage: "check-tunnel-limit",
1205 + hostname: requestedHostname,
1206 + tunnelName: requestedTunnelName,
1207 + cause,
1208 + }),
1209 + ),
1210 + }),
1211 + );
1212 const { hostname, tunnelName } = allocation;
1213
1214 if (input.reuseReadyTunnel === true) {
infra/relay/src/environments/ManagedTunnelLimits.test.ts new
+115
@@ -0,0 +1,115 @@
1 +import { describe, expect, it } from "@effect/vitest";
2 +import * as Context from "effect/Context";
3 +import * as Effect from "effect/Effect";
4 +import * as Layer from "effect/Layer";
5 +import * as Semaphore from "effect/Semaphore";
6 +import { PgDialect } from "drizzle-orm/pg-core";
7 +
8 +import * as RelayDb from "../db.ts";
9 +import { relayManagedEndpointAllocations } from "../persistence/schema.ts";
10 +import * as ManagedTunnelLimits from "./ManagedTunnelLimits.ts";
11 +
12 +class FakeTransaction extends Context.Service<FakeTransaction, { holdsLock: boolean }>()(
13 + "t3code-relay/environments/ManagedTunnelLimits.test/FakeTransaction",
14 +) {}
15 +
16 +// Models Postgres just enough: the advisory lock only works inside a transaction and is held until it ends.
17 +function makeFakeDb(reserved: ReadonlyArray<unknown>, countConditions: Array<unknown> = []) {
18 + const userLock = Semaphore.makeUnsafe(1);
19 + return {
20 + $client: {
21 + withTransaction: <A, E, R>(effect: Effect.Effect<A, E, R>) => {
22 + const transaction = { holdsLock: false };
23 + return effect.pipe(
24 + Effect.provideService(FakeTransaction, transaction),
25 + Effect.ensuring(
26 + Effect.suspend(() => (transaction.holdsLock ? userLock.release(1) : Effect.void)),
27 + ),
28 + );
29 + },
30 + },
31 + execute: () =>
32 + Effect.gen(function* () {
33 + const transaction = yield* Effect.serviceOption(FakeTransaction);
34 + if (transaction._tag === "None") {
35 + return yield* Effect.die("advisory lock taken outside a transaction");
36 + }
37 + yield* userLock.take(1);
38 + transaction.value.holdsLock = true;
39 + return [];
40 + }),
41 + select: () => ({
42 + from: (table: unknown) => ({
43 + where: (condition: unknown) => {
44 + if (table !== relayManagedEndpointAllocations) {
45 + return { limit: () => Effect.succeed([]) };
46 + }
47 + countConditions.push(condition);
48 + return Effect.yieldNow.pipe(Effect.as([{ activeTunnels: reserved.length }]));
49 + },
50 + }),
51 + }),
52 + } as unknown as RelayDb.RelayDb["Service"];
53 +}
54 +
55 +describe("ManagedTunnelLimits", () => {
56 + it.effect("rejects concurrent reservations past the limit", () => {
57 + const reserved: Array<string> = [];
58 + const attempts = ManagedTunnelLimits.DEFAULT_MANAGED_TUNNEL_LIMIT + 2;
59 +
60 + return Effect.gen(function* () {
61 + const limits = yield* ManagedTunnelLimits.ManagedTunnelLimits;
62 + const results = yield* Effect.forEach(
63 + Array.from({ length: attempts }, (_, index) => `environment-${index}`),
64 + (environmentId) =>
65 + limits
66 + .withCapacity(
67 + { userId: "user-1", environmentId },
68 + Effect.yieldNow.pipe(Effect.andThen(Effect.sync(() => reserved.push(environmentId)))),
69 + )
70 + .pipe(Effect.result),
71 + { concurrency: "unbounded" },
72 + );
73 +
74 + expect(reserved).toHaveLength(ManagedTunnelLimits.DEFAULT_MANAGED_TUNNEL_LIMIT);
75 + expect(
76 + results.filter(
77 + (result) =>
78 + result._tag === "Failure" && result.failure._tag === "ManagedTunnelLimitExceeded",
79 + ),
80 + ).toHaveLength(2);
81 + }).pipe(
82 + Effect.provide(
83 + ManagedTunnelLimits.layer.pipe(
84 + Layer.provide(Layer.succeed(RelayDb.RelayDb, makeFakeDb(reserved))),
85 + ),
86 + ),
87 + );
88 + });
89 +
90 + it.effect("pending reservations stop counting once released or withdrawn", () => {
91 + const countConditions: Array<unknown> = [];
92 +
93 + return Effect.gen(function* () {
94 + const limits = yield* ManagedTunnelLimits.ManagedTunnelLimits;
95 + yield* limits.withCapacity({ userId: "user-1", environmentId: "environment-1" }, Effect.void);
96 +
97 + expect(countConditions).toHaveLength(1);
98 + const query = new PgDialect().sqlToQuery(countConditions[0] as never);
99 + // claimRelease writes updatedAt = tunnelReleasedAt; a later reserve moves updatedAt past it.
100 + expect(query.sql).toContain(
101 + '"relay_managed_endpoint_allocations"."updated_at" > "relay_managed_endpoint_allocations"."tunnel_released_at"',
102 + );
103 + // A link revoked or made publish-only after the last reservation no longer holds a slot.
104 + expect(query.sql).toContain(
105 + '"relay_environment_links"."updated_at" >= "relay_managed_endpoint_allocations"."updated_at"',
106 + );
107 + }).pipe(
108 + Effect.provide(
109 + ManagedTunnelLimits.layer.pipe(
110 + Layer.provide(Layer.succeed(RelayDb.RelayDb, makeFakeDb([], countConditions))),
111 + ),
112 + ),
113 + );
114 + });
115 +});
infra/relay/src/environments/ManagedTunnelLimits.ts
+133 -60
@@ -1,8 +1,11 @@
1 -import { and, count, eq, isNull, ne } from "drizzle-orm";
1 +import { and, count, eq, exists, gt, gte, isNotNull, isNull, ne, not, or, sql } from "drizzle-orm";
2 +import { QueryBuilder } from "drizzle-orm/pg-core";
3 import * as Context from "effect/Context";
4 +import * as DateTime from "effect/DateTime";
5 import * as Effect from "effect/Effect";
6 import * as Layer from "effect/Layer";
7 import * as Schema from "effect/Schema";
8 +import * as SqlError from "effect/sql/SqlError";
9
10 import * as RelayDb from "../db.ts";
11 import {
@@ -11,12 +14,14 @@ import {
14 relayManagedTunnelLimits,
15 } from "../persistence/schema.ts";
16
14 -const DEFAULT_MANAGED_TUNNEL_LIMIT = 3;
17 +export const DEFAULT_MANAGED_TUNNEL_LIMIT = 3;
18 +// Reserve-to-tunnel takes two Cloudflare calls capped at 8 seconds each; a reservation older than this was abandoned.
19 +const MANAGED_TUNNEL_PENDING_WINDOW_MINUTES = 15;
20
21 export class ManagedTunnelLimitPersistenceError extends Schema.TaggedError<ManagedTunnelLimitPersistenceError>()(
22 "ManagedTunnelLimitPersistenceError",
23 {
19 - operation: Schema.Literals(["load-limit", "count-tunnels"]),
24 + operation: Schema.Literals(["transaction", "lock-user", "load-limit", "count-tunnels"]),
25 userId: Schema.String,
26 cause: Schema.Defect(),
27 },
@@ -43,10 +48,11 @@ export class ManagedTunnelLimitExceeded extends Schema.TaggedError<ManagedTunnel
48 export class ManagedTunnelLimits extends Context.Service<
49 ManagedTunnelLimits,
50 {
46 - readonly ensureCapacity: (input: {
47 - readonly userId: string;
48 - readonly environmentId: string;
49 - }) => Effect.Effect<void, ManagedTunnelLimitExceeded | ManagedTunnelLimitPersistenceError>;
51 + /** Runs `reserve` only while the user is under their limit, serialized per user in one transaction. */
52 + readonly withCapacity: <A, E, R>(
53 + input: { readonly userId: string; readonly environmentId: string },
54 + reserve: Effect.Effect<A, E, R>,
55 + ) => Effect.Effect<A, E | ManagedTunnelLimitExceeded | ManagedTunnelLimitPersistenceError, R>;
56 }
57 >()("t3code-relay/environments/ManagedTunnelLimits") {}
58
@@ -54,63 +60,130 @@ const make = Effect.gen(function* () {
60 const db = yield* RelayDb.RelayDb;
61
62 return ManagedTunnelLimits.of({
57 - ensureCapacity: Effect.fn("relay.managed_tunnel_limits.ensure_capacity")(function* (input) {
58 - const overrides = yield* db
59 - .select({ maxTunnels: relayManagedTunnelLimits.maxTunnels })
60 - .from(relayManagedTunnelLimits)
61 - .where(eq(relayManagedTunnelLimits.userId, input.userId))
62 - .limit(1)
63 - .pipe(
64 - Effect.mapError(
65 - (cause) =>
66 - new ManagedTunnelLimitPersistenceError({
67 - operation: "load-limit",
68 - userId: input.userId,
69 - cause,
70 - }),
71 - ),
63 + withCapacity: Effect.fn("relay.managed_tunnel_limits.with_capacity")(function* <A, E, R>(
64 + input: { readonly userId: string; readonly environmentId: string },
65 + reserve: Effect.Effect<A, E, R>,
66 + ): Effect.fn.Return<A, E | ManagedTunnelLimitExceeded | ManagedTunnelLimitPersistenceError, R> {
67 + const checkAndReserve = Effect.gen(function* () {
68 + // Held until commit, so the count below sees every reservation that took the lock first.
69 + yield* db
70 + .execute(
71 + sql`select pg_advisory_xact_lock(hashtextextended(${`managed-tunnel-limit:${input.userId}`}, 0))`,
72 + )
73 + .pipe(
74 + Effect.mapError(
75 + (cause) =>
76 + new ManagedTunnelLimitPersistenceError({
77 + operation: "lock-user",
78 + userId: input.userId,
79 + cause,
80 + }),
81 + ),
82 + );
83 +
84 + const overrides = yield* db
85 + .select({ maxTunnels: relayManagedTunnelLimits.maxTunnels })
86 + .from(relayManagedTunnelLimits)
87 + .where(eq(relayManagedTunnelLimits.userId, input.userId))
88 + .limit(1)
89 + .pipe(
90 + Effect.mapError(
91 + (cause) =>
92 + new ManagedTunnelLimitPersistenceError({
93 + operation: "load-limit",
94 + userId: input.userId,
95 + cause,
96 + }),
97 + ),
98 + );
99 + const maxTunnels = overrides[0]?.maxTunnels ?? DEFAULT_MANAGED_TUNNEL_LIMIT;
100 +
101 + const pendingCutoff = DateTime.formatIso(
102 + DateTime.subtract(yield* DateTime.now, {
103 + minutes: MANAGED_TUNNEL_PENDING_WINDOW_MINUTES,
104 + }),
105 );
73 - const maxTunnels = overrides[0]?.maxTunnels ?? DEFAULT_MANAGED_TUNNEL_LIMIT;
106 + // Counts allocations, not links: a link is written only after its tunnel exists.
107 + // Excludes this environment so relinking at the limit stays idempotent.
108 + const allocation = relayManagedEndpointAllocations;
109 + const withdrawnLink = (since?: typeof allocation.updatedAt) =>
110 + exists(
111 + new QueryBuilder()
112 + .select({ userId: relayEnvironmentLinks.userId })
113 + .from(relayEnvironmentLinks)
114 + .where(
115 + and(
116 + eq(relayEnvironmentLinks.userId, allocation.userId),
117 + eq(relayEnvironmentLinks.environmentId, allocation.environmentId),
118 + or(
119 + isNotNull(relayEnvironmentLinks.revokedAt),
120 + eq(relayEnvironmentLinks.managedTunnelsEnabled, false),
121 + ),
122 + since === undefined ? undefined : gte(relayEnvironmentLinks.updatedAt, since),
123 + ),
124 + ),
125 + );
126 + const counted = yield* db
127 + .select({ activeTunnels: count() })
128 + .from(allocation)
129 + .where(
130 + and(
131 + eq(allocation.userId, input.userId),
132 + ne(allocation.environmentId, input.environmentId),
133 + or(
134 + // Reserved but not yet holding a tunnel. Release writes updatedAt = tunnelReleasedAt and
135 + // a relink reserves before rewriting its link, so only a newer reservation counts here.
136 + and(
137 + gt(allocation.updatedAt, pendingCutoff),
138 + or(
139 + isNull(allocation.tunnelReleasedAt),
140 + gt(allocation.updatedAt, allocation.tunnelReleasedAt),
141 + ),
142 + not(withdrawnLink(allocation.updatedAt)),
143 + ),
144 + and(
145 + isNotNull(allocation.tunnelId),
146 + isNull(allocation.tunnelReleasedAt),
147 + not(withdrawnLink()),
148 + ),
149 + ),
150 + ),
151 + )
152 + .pipe(
153 + Effect.mapError(
154 + (cause) =>
155 + new ManagedTunnelLimitPersistenceError({
156 + operation: "count-tunnels",
157 + userId: input.userId,
158 + cause,
159 + }),
160 + ),
161 + );
162 + const activeTunnels = counted[0]?.activeTunnels ?? 0;
163
75 - // Exclude this environment so relinking remains idempotent at the account limit.
76 - const counted = yield* db
77 - .select({ activeTunnels: count() })
78 - .from(relayManagedEndpointAllocations)
79 - .innerJoin(
80 - relayEnvironmentLinks,
81 - and(
82 - eq(relayEnvironmentLinks.userId, relayManagedEndpointAllocations.userId),
83 - eq(relayEnvironmentLinks.environmentId, relayManagedEndpointAllocations.environmentId),
84 - ),
85 - )
86 - .where(
87 - and(
88 - eq(relayManagedEndpointAllocations.userId, input.userId),
89 - ne(relayManagedEndpointAllocations.environmentId, input.environmentId),
90 - isNull(relayEnvironmentLinks.revokedAt),
91 - eq(relayEnvironmentLinks.managedTunnelsEnabled, true),
92 - ),
93 - )
94 - .pipe(
95 - Effect.mapError(
96 - (cause) =>
97 - new ManagedTunnelLimitPersistenceError({
98 - operation: "count-tunnels",
164 + if (activeTunnels >= maxTunnels) {
165 + return yield* new ManagedTunnelLimitExceeded({
166 + userId: input.userId,
167 + environmentId: input.environmentId,
168 + maxTunnels,
169 + activeTunnels,
170 + });
171 + }
172 + return yield* reserve;
173 + });
174 +
175 + // Statements inside map their own SqlErrors, so any left here come from opening or committing the transaction.
176 + return yield* db.$client.withTransaction(checkAndReserve).pipe(
177 + Effect.mapError((cause) =>
178 + SqlError.isSqlError(cause)
179 + ? new ManagedTunnelLimitPersistenceError({
180 + operation: "transaction",
181 userId: input.userId,
182 cause,
101 - }),
102 - ),
103 - );
104 - const activeTunnels = counted[0]?.activeTunnels ?? 0;
105 -
106 - if (activeTunnels >= maxTunnels) {
107 - return yield* new ManagedTunnelLimitExceeded({
108 - userId: input.userId,
109 - environmentId: input.environmentId,
110 - maxTunnels,
111 - activeTunnels,
112 - });
113 - }
183 + })
184 + : cause,
185 + ),
186 + );
187 }),
188 });
189 });