Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 269 lines 11.2 KB
Raw
1 import * as BunHttpServer from '@effect/platform-bun/BunHttpServer';
2 import { Effect, FileSystem, Option, Path, Schema, Stdio } from 'effect';
3 import { Digest } from 'effect/crypto';
4 import { Base64 } from 'effect/encoding';
5 import { HttpClient, HttpServer, HttpServerRequest, HttpServerResponse } from 'effect/http';
6 import { PackedManifest, tarballName } from './candidates';
7 import { readVerifiedCandidate } from './consumers';
8 import { NPM_REGISTRY } from './lanes';
9 import {
10 ensure,
11 fail,
12 formatJson,
13 readJson,
14 run,
15 runMain,
16 withoutReleaseTokens,
17 writeFile,
18 } from './runtime';
19 import { BATCH_PATH, ReleaseBatch } from './source';
20
21 type PlannedPackage = Pick<ReleaseBatch['plan']['releases'][number], 'name' | 'newVersion'>;
22
23 export interface CandidatePackage {
24 readonly manifest: typeof PackedManifest.Type;
25 readonly bytes: Uint8Array;
26 readonly integrity: string;
27 }
28
29 /**
30 * Reads the planned packages from a downloaded TypeScript candidate. Every file is checked
31 * against the candidate manifest first, so the registry never serves bytes the candidate job did
32 * not build.
33 */
34 export const loadCandidatePackages = (
35 candidates: string,
36 releases: ReadonlyArray<PlannedPackage>
37 ) =>
38 Effect.gen(function* () {
39 const digest = yield* Digest.Digest;
40 const { manifest, files } = yield* readVerifiedCandidate(candidates);
41 yield* ensure(manifest.language === 'typescript', 'Expected the TypeScript candidate');
42 return yield* Effect.forEach(releases, release =>
43 Effect.gen(function* () {
44 const file = tarballName(release.name, release.newVersion);
45 const bytes = files.get(file);
46 if (!bytes) return yield* fail(`Missing candidate tarball: ${file}`);
47 const packed = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(PackedManifest))(
48 yield* run('tar', ['-xOf', `${candidates}/${file}`, 'package/package.json'], {
49 env: withoutReleaseTokens,
50 })
51 );
52 yield* ensure(
53 packed.name === release.name && packed.version === release.newVersion,
54 `Candidate ${file} does not contain ${release.name}@${release.newVersion}`
55 );
56 const integrity = `sha512-${Base64.encode(yield* digest.digest(Digest.sha512, bytes))}`;
57 return { manifest: packed, bytes, integrity } satisfies CandidatePackage;
58 })
59 );
60 });
61
62 /** The npm package a registry path names: `@scope/name/-/name-1.0.0.tgz` names `@scope/name`. */
63 const packageOf = (path: string) => {
64 const [first = '', second = ''] = path.split('/');
65 return first.startsWith('@') ? `${first}/${second}` : first;
66 };
67
68 /**
69 * Starts an npm registry on localhost for the current scope. It serves the planned packages from
70 * their candidate tarballs, at the candidate version only, and proxies every other package to
71 * `upstream`. A planned package never falls through to `upstream`, so no copy can resolve to a
72 * published version. Returns the registry URL.
73 */
74 export const serveCandidateRegistry = (
75 packages: ReadonlyArray<CandidatePackage>,
76 upstream: string = NPM_REGISTRY
77 ) =>
78 Effect.gen(function* () {
79 const client = yield* HttpClient.HttpClient;
80 const server = yield* BunHttpServer.make({ port: 0, hostname: '127.0.0.1' });
81 const origin = HttpServer.formatAddress(server.address);
82 const packuments = new Map<string, unknown>();
83 const tarballs = new Map<string, Uint8Array>();
84 for (const { manifest, bytes, integrity } of packages) {
85 const tarball = `${manifest.name}/-/${manifest.name.split('/').at(-1)}-${manifest.version}.tgz`;
86 tarballs.set(tarball, bytes);
87 packuments.set(manifest.name, {
88 name: manifest.name,
89 'dist-tags': { latest: manifest.version },
90 versions: {
91 [manifest.version]: { ...manifest, dist: { tarball: `${origin}/${tarball}`, integrity } },
92 },
93 });
94 }
95 yield* server.serve(
96 Effect.gen(function* () {
97 const request = yield* HttpServerRequest.HttpServerRequest;
98 const url = new URL(request.url, origin);
99 // Clients spell a scoped packument as `@scope%2fname` or `@scope/name`.
100 const path = decodeURIComponent(url.pathname).slice(1);
101 const packument = packuments.get(path);
102 if (packument) return HttpServerResponse.jsonUnsafe(packument);
103 const tarball = tarballs.get(path);
104 if (tarball)
105 return HttpServerResponse.uint8Array(tarball, {
106 contentType: 'application/octet-stream',
107 });
108 if (request.method !== 'GET' || packuments.has(packageOf(path)))
109 return HttpServerResponse.empty({ status: 404 });
110 const response = yield* client.get(`${upstream}${url.pathname}${url.search}`, {
111 headers: { accept: request.headers.accept ?? '*/*' },
112 });
113 return HttpServerResponse.uint8Array(new Uint8Array(yield* response.arrayBuffer), {
114 status: response.status,
115 contentType: response.headers['content-type'],
116 });
117 })
118 );
119 return origin;
120 });
121
122 const DEPENDENCY_FIELDS = ['dependencies', 'devDependencies', 'optionalDependencies'] as const;
123 const Dependencies = Schema.optionalKey(Schema.Record(Schema.String, Schema.String));
124 const DocsManifest = Schema.StructWithRest(
125 Schema.Struct({
126 dependencies: Dependencies,
127 devDependencies: Dependencies,
128 optionalDependencies: Dependencies,
129 overrides: Schema.optionalKey(Schema.Record(Schema.String, Schema.Unknown)),
130 }),
131 [Schema.Record(Schema.String, Schema.Unknown)]
132 );
133
134 /**
135 * Pins the planned packages to their candidate versions in this checkout's docs manifest, as
136 * direct dependencies and as overrides for every nested copy. The committed docs pins name
137 * published versions; this job-local edit is never committed. Returns the planned packages the
138 * docs depend on directly.
139 */
140 export const pinCandidateVersions = (
141 manifestPath: string,
142 releases: ReadonlyArray<PlannedPackage>
143 ) =>
144 Effect.gen(function* () {
145 const manifest = yield* readJson(DocsManifest, manifestPath);
146 const pins = Object.fromEntries(releases.map(release => [release.name, release.newVersion]));
147 const direct = releases
148 .map(release => release.name)
149 .filter(name => DEPENDENCY_FIELDS.some(field => manifest[field]?.[name] !== undefined));
150 const pinned: Record<string, unknown> = {
151 ...manifest,
152 overrides: { ...manifest.overrides, ...pins },
153 };
154 for (const field of DEPENDENCY_FIELDS) {
155 const dependencies = manifest[field];
156 if (!dependencies) continue;
157 pinned[field] = Object.fromEntries(
158 Object.entries(dependencies).map(([name, range]) => [name, pins[name] ?? range])
159 );
160 }
161 yield* writeFile(manifestPath, formatJson(pinned));
162 return direct;
163 });
164
165 const Versioned = Schema.Struct({ version: Schema.String });
166
167 const installedCopies = (nodeModules: string, names: ReadonlySet<string>) =>
168 Effect.gen(function* () {
169 const fs = yield* FileSystem.FileSystem;
170 const path = yield* Path.Path;
171 const copies: { name: string; directory: string; version: string }[] = [];
172 const pending = [nodeModules];
173 for (let directory = pending.pop(); directory !== undefined; directory = pending.pop()) {
174 const info = yield* Effect.option(fs.stat(directory));
175 if (Option.isNone(info) || info.value.type !== 'Directory') continue;
176 for (const entry of yield* fs.readDirectory(directory)) {
177 // The isolated linker keeps each package under `.bun/<name>@<version>/node_modules`.
178 if (entry === '.bun') {
179 for (const store of yield* fs.readDirectory(path.join(directory, entry)))
180 pending.push(path.join(directory, entry, store, 'node_modules'));
181 continue;
182 }
183 const packageNames = entry.startsWith('@')
184 ? (yield* fs.readDirectory(path.join(directory, entry))).map(
185 scoped => `${entry}/${scoped}`
186 )
187 : [entry];
188 for (const name of packageNames) {
189 const packageDirectory = path.join(directory, name);
190 if (names.has(name))
191 copies.push({
192 name,
193 directory: packageDirectory,
194 version: (yield* readJson(Versioned, path.join(packageDirectory, 'package.json')))
195 .version,
196 });
197 pending.push(path.join(packageDirectory, 'node_modules'));
198 }
199 }
200 }
201 return copies;
202 });
203
204 /**
205 * Fails unless every installed copy of a planned package is its candidate version and each
206 * planned package in `required` is installed at all.
207 */
208 export const assertCandidateVersions = (
209 nodeModules: string,
210 releases: ReadonlyArray<PlannedPackage>,
211 required: ReadonlyArray<string>
212 ) =>
213 Effect.gen(function* () {
214 const versions = new Map(releases.map(release => [release.name, release.newVersion]));
215 const copies = yield* installedCopies(nodeModules, new Set(versions.keys()));
216 const problems = [
217 ...required
218 .filter(name => !copies.some(copy => copy.name === name))
219 .map(name => `${name} is not installed`),
220 ...copies
221 .filter(copy => copy.version !== versions.get(copy.name))
222 .map(copy => `${copy.directory} is ${copy.version}, not ${versions.get(copy.name)}`),
223 ];
224 yield* ensure(
225 problems.length === 0,
226 `The docs did not install the candidate packages:\n${problems.join('\n')}`
227 );
228 });
229
230 const bun = (docs: string, args: ReadonlyArray<string>, env: Record<string, string> = {}) =>
231 run('bun', args, { cwd: docs, env: { ...withoutReleaseTokens, ...env }, output: 'inherit' });
232
233 /**
234 * Installs the docs dependencies for a stable release PR. Without a TypeScript release, the
235 * committed lock is installed from npm as is. Otherwise the planned packages come from the
236 * candidate through a job-local registry, every other package through its npm proxy, and every
237 * installed copy of a planned package must be its candidate version.
238 */
239 export const installCandidateDocs = (docs: string, batch: ReleaseBatch, candidates: string) =>
240 Effect.scoped(
241 Effect.gen(function* () {
242 const fs = yield* FileSystem.FileSystem;
243 const path = yield* Path.Path;
244 const npm = batch.plan.releases.filter(release => release.language === 'typescript');
245 if (npm.length === 0) return yield* bun(docs, ['install', '--frozen-lockfile']);
246 const packages = yield* loadCandidatePackages(candidates, npm);
247 const registry = yield* serveCandidateRegistry(packages);
248 const direct = yield* pinCandidateVersions(path.join(docs, 'package.json'), npm);
249 // A fresh cache, so no planned package is reused from an earlier install.
250 const cache = yield* fs.makeTempDirectoryScoped({ prefix: 'sdk-candidate-docs-cache-' });
251 yield* bun(docs, ['install'], {
252 NPM_CONFIG_REGISTRY: registry,
253 BUN_INSTALL_CACHE_DIR: cache,
254 });
255 yield* assertCandidateVersions(path.join(docs, 'node_modules'), npm, direct);
256 })
257 );
258
259 /** `candidate-registry.ts <candidate-dir>` installs the docs against a stable release's candidate. */
260 const main = Effect.gen(function* () {
261 const path = yield* Path.Path;
262 const [candidates] = yield* Schema.decodeUnknownEffect(Schema.Tuple([Schema.String]))(
263 yield* (yield* Stdio.Stdio).args
264 );
265 const batch = yield* readJson(ReleaseBatch, BATCH_PATH);
266 yield* installCandidateDocs(path.resolve('docs'), batch, path.resolve(candidates));
267 });
268
269 if (import.meta.main) runMain(main);