@devries-cameron20 / t3code Mirror · read-only / commits / e1af0b2438
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 12:00 UTC

perf(provider-core): provider instances share one latest-version lookup (#23763)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: maria-rcks <maria@kuuro.net>

Victor Vogel committed Oct 30, 2026 at 15:59 UTC e1af0b24381fc962080d871bdc962abb2426af52
2 files changed +179 -16
packages/provider-core/src/server/ProviderLatestVersions.test.ts new
+136
@@ -0,0 +1,136 @@
1 +import { describe, expect, it } from "@effect/vitest";
2 +import * as Deferred from "effect/Deferred";
3 +import * as Effect from "effect/Effect";
4 +import * as Exit from "effect/Exit";
5 +import * as Fiber from "effect/Fiber";
6 +
7 +import * as ProviderLatestVersions from "./ProviderLatestVersions.ts";
8 +
9 +describe("ProviderLatestVersions", () => {
10 + it.effect("concurrent misses for one key share a single lookup", () =>
11 + Effect.gen(function* () {
12 + const latestVersions = yield* ProviderLatestVersions.make();
13 + const release = yield* Deferred.make<void>();
14 + let lookups = 0;
15 + const lookup = Effect.suspend(() => {
16 + lookups += 1;
17 + return Deferred.await(release).pipe(Effect.as("2.0.0"));
18 + });
19 +
20 + const callers = yield* Effect.forEach([1, 2, 3], () =>
21 + Effect.forkChild(latestVersions.cached("@example/cli", lookup), {
22 + startImmediately: true,
23 + }),
24 + );
25 + yield* Deferred.succeed(release, undefined);
26 + expect(yield* Effect.forEach(callers, Fiber.join)).toEqual(["2.0.0", "2.0.0", "2.0.0"]);
27 + expect(lookups).toBe(1);
28 + }),
29 + );
30 +
31 + it.effect("a waiting caller looks up again when the shared lookup is interrupted", () =>
32 + Effect.gen(function* () {
33 + const latestVersions = yield* ProviderLatestVersions.make();
34 + let lookups = 0;
35 + const lookup = Effect.suspend(() => {
36 + lookups += 1;
37 + return lookups === 1 ? Effect.never : Effect.succeed("2.0.0");
38 + });
39 +
40 + const owner = yield* Effect.forkChild(latestVersions.cached("@example/cli", lookup), {
41 + startImmediately: true,
42 + });
43 + const waiter = yield* Effect.forkChild(latestVersions.cached("@example/cli", lookup), {
44 + startImmediately: true,
45 + });
46 + expect(lookups).toBe(1);
47 + yield* Fiber.interrupt(owner);
48 + expect(yield* Fiber.join(waiter)).toBe("2.0.0");
49 + expect(yield* latestVersions.cached("@example/cli", lookup)).toBe("2.0.0");
50 + expect(lookups).toBe(2);
51 + }),
52 + );
53 +
54 + it.effect("a caller interrupted while it claims the lookup frees the key", () =>
55 + Effect.gen(function* () {
56 + const latestVersions = yield* ProviderLatestVersions.make();
57 + let lookups = 0;
58 + const lookup = Effect.sync(() => {
59 + lookups += 1;
60 + return "2.0.0";
61 + });
62 + // `cached` reads `fresh` in the same step that claims the key, so the
63 + // interrupt arrives after the claim and before the lookup starts.
64 + const interruptOnClaim = {
65 + get fresh() {
66 + Fiber.getCurrent()?.interruptUnsafe();
67 + return false;
68 + },
69 + };
70 +
71 + const owner = yield* Effect.forkChild(
72 + latestVersions.cached("@example/cli", lookup, interruptOnClaim),
73 + { startImmediately: true },
74 + );
75 + expect(Exit.hasInterrupts(yield* Fiber.await(owner))).toBe(true);
76 + expect(lookups).toBe(0);
77 + const next = yield* Effect.forkChild(latestVersions.cached("@example/cli", lookup), {
78 + startImmediately: true,
79 + });
80 + expect(lookups).toBe(1);
81 + expect(yield* Fiber.join(next)).toBe("2.0.0");
82 + }),
83 + );
84 + it.effect("a waiting caller retries with its own lookup after the owner fails", () =>
85 + Effect.gen(function* () {
86 + const latestVersions = yield* ProviderLatestVersions.make();
87 + const release = yield* Deferred.make<void>();
88 + const owner = yield* Effect.forkChild(
89 + latestVersions.cached(
90 + "cli",
91 + Deferred.await(release).pipe(Effect.andThen(Effect.fail("offline"))),
92 + ),
93 + { startImmediately: true },
94 + );
95 + let retries = 0;
96 + const waiter = yield* Effect.forkChild(
97 + latestVersions.cached(
98 + "cli",
99 + Effect.sync(() => {
100 + retries += 1;
101 + return "2.0.0";
102 + }),
103 + ),
104 + { startImmediately: true },
105 + );
106 + expect(retries).toBe(0);
107 + yield* Deferred.succeed(release, undefined);
108 + expect(yield* Effect.flip(Fiber.join(owner))).toBe("offline");
109 + expect(yield* Fiber.join(waiter)).toBe("2.0.0");
110 + expect(retries).toBe(1);
111 + }),
112 + );
113 +
114 + it.effect("interrupting a waiter leaves the shared lookup running", () =>
115 + Effect.gen(function* () {
116 + const latestVersions = yield* ProviderLatestVersions.make();
117 + const release = yield* Deferred.make<void>();
118 + let lookups = 0;
119 + const lookup = Effect.suspend(() => {
120 + lookups += 1;
121 + return Deferred.await(release).pipe(Effect.as("2.0.0"));
122 + });
123 + const owner = yield* Effect.forkChild(latestVersions.cached("cli", lookup), {
124 + startImmediately: true,
125 + });
126 + const waiter = yield* Effect.forkChild(latestVersions.cached("cli", lookup), {
127 + startImmediately: true,
128 + });
129 + yield* Fiber.interrupt(waiter);
130 + yield* Deferred.succeed(release, undefined);
131 + expect(yield* Fiber.join(owner)).toBe("2.0.0");
132 + expect(yield* latestVersions.cached("cli", lookup)).toBe("2.0.0");
133 + expect(lookups).toBe(1);
134 + }),
135 + );
136 +});
packages/provider-core/src/server/ProviderLatestVersions.ts
+43 -16
@@ -1,8 +1,11 @@
1 import * as Context from "effect/Context";
2 import * as DateTime from "effect/DateTime";
3 +import * as Deferred from "effect/Deferred";
4 import * as Duration from "effect/Duration";
5 import * as Effect from "effect/Effect";
6 +import * as Exit from "effect/Exit";
7 import * as Layer from "effect/Layer";
8 +import * as Option from "effect/Option";
9 import * as Ref from "effect/Ref";
10
11 const FOUND_TTL = Duration.hours(1);
@@ -11,6 +14,7 @@ const MISSING_TTL = Duration.minutes(1);
14 interface Entry {
15 readonly expiresAt: number;
16 readonly version: string | null;
17 + readonly inFlight?: Deferred.Deferred<Option.Option<string | null>>;
18 }
19
20 export class ProviderLatestVersions extends Context.Service<
@@ -33,23 +37,46 @@ export const make = Effect.fn("ProviderLatestVersions.make")(function* (
37 Array.from(seed, ([key, version]) => [key, { version, expiresAt: Number.MAX_SAFE_INTEGER }]),
38 ),
39 );
40 + const cached: ProviderLatestVersions["Service"]["cached"] = Effect.fn(
41 + "ProviderLatestVersions.cached",
42 + )(function* (key, lookup, options) {
43 + const now = DateTime.toEpochMillis(yield* DateTime.now);
44 + const inFlight = yield* Deferred.make<Option.Option<string | null>>();
45 + // Replacing or invalidating this entry retires any lookup still in flight.
46 + const pending: Entry = { version: null, expiresAt: 0, inFlight };
47 + // Claim and cleanup registration must be atomic with respect to interruption.
48 + return yield* Effect.acquireUseRelease(
49 + Ref.modify(entries, (current) => {
50 + const entry = current.get(key);
51 + return !options?.fresh && entry && (entry.expiresAt > now || entry.inFlight)
52 + ? [entry, current]
53 + : [undefined, new Map(current).set(key, pending)];
54 + }),
55 + (existing) =>
56 + Effect.gen(function* () {
57 + if (!existing) return yield* lookup;
58 + if (!existing.inFlight) return existing.version;
59 + const shared = yield* Deferred.await(existing.inFlight);
60 + return Option.isSome(shared) ? shared.value : yield* cached(key, lookup, options);
61 + }),
62 + (existing, exit) => {
63 + if (existing) return Effect.void;
64 + const version = Exit.isSuccess(exit) ? Option.some(exit.value) : Option.none();
65 + return Ref.update(entries, (current) => {
66 + if (current.get(key) !== pending) return current;
67 + const next = new Map(current);
68 + if (Option.isNone(version)) {
69 + next.delete(key);
70 + return next;
71 + }
72 + const ttl = version.value === null ? MISSING_TTL : FOUND_TTL;
73 + return next.set(key, { version: version.value, expiresAt: now + Duration.toMillis(ttl) });
74 + }).pipe(Effect.andThen(Deferred.succeed(inFlight, version)), Effect.asVoid);
75 + },
76 + );
77 + });
78 return ProviderLatestVersions.of({
37 - cached: Effect.fn("ProviderLatestVersions.cached")(function* (key, lookup, options) {
38 - const now = DateTime.toEpochMillis(yield* DateTime.now);
39 - const existing = (yield* Ref.get(entries)).get(key);
40 - if (!options?.fresh && existing && existing.expiresAt > now) return existing.version;
41 - // Replacing or invalidating this entry retires any lookup still in flight.
42 - const pending = { version: null, expiresAt: 0 };
43 - yield* Ref.update(entries, (current) => new Map(current).set(key, pending));
44 - const version = yield* lookup;
45 - const ttl = version === null ? MISSING_TTL : FOUND_TTL;
46 - yield* Ref.update(entries, (current) =>
47 - current.get(key) === pending
48 - ? new Map(current).set(key, { version, expiresAt: now + Duration.toMillis(ttl) })
49 - : current,
50 - );
51 - return version;
52 - }),
79 + cached,
80 invalidate: Effect.fn("ProviderLatestVersions.invalidate")(function (key) {
81 return Ref.update(entries, (current) => {
82 const next = new Map(current);