Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | import * as BunHttpServer from '@effect/platform-bun/BunHttpServer'; |
| 2 | import { Effect, FileSystem, Option, Path, Schema, Stdio } from 'effect'; |
| 3 | import { Digest } from 'effect/crypto'; |
| 4 | import { Base64 } from 'effect/encoding'; |
| 5 | import { HttpClient, HttpServer, HttpServerRequest, HttpServerResponse } from 'effect/http'; |
| 6 | import { PackedManifest, tarballName } from './candidates'; |
| 7 | import { readVerifiedCandidate } from './consumers'; |
| 8 | import { NPM_REGISTRY } from './lanes'; |
| 9 | import { |
| 10 | ensure, |
| 11 | fail, |
| 12 | formatJson, |
| 13 | readJson, |
| 14 | run, |
| 15 | runMain, |
| 16 | withoutReleaseTokens, |
| 17 | writeFile, |
| 18 | } from './runtime'; |
| 19 | import { BATCH_PATH, ReleaseBatch } from './source'; |
| 20 | |
| 21 | type PlannedPackage = Pick<ReleaseBatch['plan']['releases'][number], 'name' | 'newVersion'>; |
| 22 | |
| 23 | export interface CandidatePackage { |
| 24 | readonly manifest: typeof PackedManifest.Type; |
| 25 | readonly bytes: Uint8Array; |
| 26 | readonly integrity: string; |
| 27 | } |
| 28 | |
| 29 | /** |
| 30 | * Reads the planned packages from a downloaded TypeScript candidate. Every file is checked |
| 31 | * against the candidate manifest first, so the registry never serves bytes the candidate job did |
| 32 | * not build. |
| 33 | */ |
| 34 | export const loadCandidatePackages = ( |
| 35 | candidates: string, |
| 36 | releases: ReadonlyArray<PlannedPackage> |
| 37 | ) => |
| 38 | Effect.gen(function* () { |
| 39 | const digest = yield* Digest.Digest; |
| 40 | const { manifest, files } = yield* readVerifiedCandidate(candidates); |
| 41 | yield* ensure(manifest.language === 'typescript', 'Expected the TypeScript candidate'); |
| 42 | return yield* Effect.forEach(releases, release => |
| 43 | Effect.gen(function* () { |
| 44 | const file = tarballName(release.name, release.newVersion); |
| 45 | const bytes = files.get(file); |
| 46 | if (!bytes) return yield* fail(`Missing candidate tarball: ${file}`); |
| 47 | const packed = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(PackedManifest))( |
| 48 | yield* run('tar', ['-xOf', `${candidates}/${file}`, 'package/package.json'], { |
| 49 | env: withoutReleaseTokens, |
| 50 | }) |
| 51 | ); |
| 52 | yield* ensure( |
| 53 | packed.name === release.name && packed.version === release.newVersion, |
| 54 | `Candidate ${file} does not contain ${release.name}@${release.newVersion}` |
| 55 | ); |
| 56 | const integrity = `sha512-${Base64.encode(yield* digest.digest(Digest.sha512, bytes))}`; |
| 57 | return { manifest: packed, bytes, integrity } satisfies CandidatePackage; |
| 58 | }) |
| 59 | ); |
| 60 | }); |
| 61 | |
| 62 | /** The npm package a registry path names: `@scope/name/-/name-1.0.0.tgz` names `@scope/name`. */ |
| 63 | const packageOf = (path: string) => { |
| 64 | const [first = '', second = ''] = path.split('/'); |
| 65 | return first.startsWith('@') ? `${first}/${second}` : first; |
| 66 | }; |
| 67 | |
| 68 | /** |
| 69 | * Starts an npm registry on localhost for the current scope. It serves the planned packages from |
| 70 | * their candidate tarballs, at the candidate version only, and proxies every other package to |
| 71 | * `upstream`. A planned package never falls through to `upstream`, so no copy can resolve to a |
| 72 | * published version. Returns the registry URL. |
| 73 | */ |
| 74 | export const serveCandidateRegistry = ( |
| 75 | packages: ReadonlyArray<CandidatePackage>, |
| 76 | upstream: string = NPM_REGISTRY |
| 77 | ) => |
| 78 | Effect.gen(function* () { |
| 79 | const client = yield* HttpClient.HttpClient; |
| 80 | const server = yield* BunHttpServer.make({ port: 0, hostname: '127.0.0.1' }); |
| 81 | const origin = HttpServer.formatAddress(server.address); |
| 82 | const packuments = new Map<string, unknown>(); |
| 83 | const tarballs = new Map<string, Uint8Array>(); |
| 84 | for (const { manifest, bytes, integrity } of packages) { |
| 85 | const tarball = `${manifest.name}/-/${manifest.name.split('/').at(-1)}-${manifest.version}.tgz`; |
| 86 | tarballs.set(tarball, bytes); |
| 87 | packuments.set(manifest.name, { |
| 88 | name: manifest.name, |
| 89 | 'dist-tags': { latest: manifest.version }, |
| 90 | versions: { |
| 91 | [manifest.version]: { ...manifest, dist: { tarball: `${origin}/${tarball}`, integrity } }, |
| 92 | }, |
| 93 | }); |
| 94 | } |
| 95 | yield* server.serve( |
| 96 | Effect.gen(function* () { |
| 97 | const request = yield* HttpServerRequest.HttpServerRequest; |
| 98 | const url = new URL(request.url, origin); |
| 99 | // Clients spell a scoped packument as `@scope%2fname` or `@scope/name`. |
| 100 | const path = decodeURIComponent(url.pathname).slice(1); |
| 101 | const packument = packuments.get(path); |
| 102 | if (packument) return HttpServerResponse.jsonUnsafe(packument); |
| 103 | const tarball = tarballs.get(path); |
| 104 | if (tarball) |
| 105 | return HttpServerResponse.uint8Array(tarball, { |
| 106 | contentType: 'application/octet-stream', |
| 107 | }); |
| 108 | if (request.method !== 'GET' || packuments.has(packageOf(path))) |
| 109 | return HttpServerResponse.empty({ status: 404 }); |
| 110 | const response = yield* client.get(`${upstream}${url.pathname}${url.search}`, { |
| 111 | headers: { accept: request.headers.accept ?? '*/*' }, |
| 112 | }); |
| 113 | return HttpServerResponse.uint8Array(new Uint8Array(yield* response.arrayBuffer), { |
| 114 | status: response.status, |
| 115 | contentType: response.headers['content-type'], |
| 116 | }); |
| 117 | }) |
| 118 | ); |
| 119 | return origin; |
| 120 | }); |
| 121 | |
| 122 | const DEPENDENCY_FIELDS = ['dependencies', 'devDependencies', 'optionalDependencies'] as const; |
| 123 | const Dependencies = Schema.optionalKey(Schema.Record(Schema.String, Schema.String)); |
| 124 | const DocsManifest = Schema.StructWithRest( |
| 125 | Schema.Struct({ |
| 126 | dependencies: Dependencies, |
| 127 | devDependencies: Dependencies, |
| 128 | optionalDependencies: Dependencies, |
| 129 | overrides: Schema.optionalKey(Schema.Record(Schema.String, Schema.Unknown)), |
| 130 | }), |
| 131 | [Schema.Record(Schema.String, Schema.Unknown)] |
| 132 | ); |
| 133 | |
| 134 | /** |
| 135 | * Pins the planned packages to their candidate versions in this checkout's docs manifest, as |
| 136 | * direct dependencies and as overrides for every nested copy. The committed docs pins name |
| 137 | * published versions; this job-local edit is never committed. Returns the planned packages the |
| 138 | * docs depend on directly. |
| 139 | */ |
| 140 | export const pinCandidateVersions = ( |
| 141 | manifestPath: string, |
| 142 | releases: ReadonlyArray<PlannedPackage> |
| 143 | ) => |
| 144 | Effect.gen(function* () { |
| 145 | const manifest = yield* readJson(DocsManifest, manifestPath); |
| 146 | const pins = Object.fromEntries(releases.map(release => [release.name, release.newVersion])); |
| 147 | const direct = releases |
| 148 | .map(release => release.name) |
| 149 | .filter(name => DEPENDENCY_FIELDS.some(field => manifest[field]?.[name] !== undefined)); |
| 150 | const pinned: Record<string, unknown> = { |
| 151 | ...manifest, |
| 152 | overrides: { ...manifest.overrides, ...pins }, |
| 153 | }; |
| 154 | for (const field of DEPENDENCY_FIELDS) { |
| 155 | const dependencies = manifest[field]; |
| 156 | if (!dependencies) continue; |
| 157 | pinned[field] = Object.fromEntries( |
| 158 | Object.entries(dependencies).map(([name, range]) => [name, pins[name] ?? range]) |
| 159 | ); |
| 160 | } |
| 161 | yield* writeFile(manifestPath, formatJson(pinned)); |
| 162 | return direct; |
| 163 | }); |
| 164 | |
| 165 | const Versioned = Schema.Struct({ version: Schema.String }); |
| 166 | |
| 167 | const installedCopies = (nodeModules: string, names: ReadonlySet<string>) => |
| 168 | Effect.gen(function* () { |
| 169 | const fs = yield* FileSystem.FileSystem; |
| 170 | const path = yield* Path.Path; |
| 171 | const copies: { name: string; directory: string; version: string }[] = []; |
| 172 | const pending = [nodeModules]; |
| 173 | for (let directory = pending.pop(); directory !== undefined; directory = pending.pop()) { |
| 174 | const info = yield* Effect.option(fs.stat(directory)); |
| 175 | if (Option.isNone(info) || info.value.type !== 'Directory') continue; |
| 176 | for (const entry of yield* fs.readDirectory(directory)) { |
| 177 | // The isolated linker keeps each package under `.bun/<name>@<version>/node_modules`. |
| 178 | if (entry === '.bun') { |
| 179 | for (const store of yield* fs.readDirectory(path.join(directory, entry))) |
| 180 | pending.push(path.join(directory, entry, store, 'node_modules')); |
| 181 | continue; |
| 182 | } |
| 183 | const packageNames = entry.startsWith('@') |
| 184 | ? (yield* fs.readDirectory(path.join(directory, entry))).map( |
| 185 | scoped => `${entry}/${scoped}` |
| 186 | ) |
| 187 | : [entry]; |
| 188 | for (const name of packageNames) { |
| 189 | const packageDirectory = path.join(directory, name); |
| 190 | if (names.has(name)) |
| 191 | copies.push({ |
| 192 | name, |
| 193 | directory: packageDirectory, |
| 194 | version: (yield* readJson(Versioned, path.join(packageDirectory, 'package.json'))) |
| 195 | .version, |
| 196 | }); |
| 197 | pending.push(path.join(packageDirectory, 'node_modules')); |
| 198 | } |
| 199 | } |
| 200 | } |
| 201 | return copies; |
| 202 | }); |
| 203 | |
| 204 | /** |
| 205 | * Fails unless every installed copy of a planned package is its candidate version and each |
| 206 | * planned package in `required` is installed at all. |
| 207 | */ |
| 208 | export const assertCandidateVersions = ( |
| 209 | nodeModules: string, |
| 210 | releases: ReadonlyArray<PlannedPackage>, |
| 211 | required: ReadonlyArray<string> |
| 212 | ) => |
| 213 | Effect.gen(function* () { |
| 214 | const versions = new Map(releases.map(release => [release.name, release.newVersion])); |
| 215 | const copies = yield* installedCopies(nodeModules, new Set(versions.keys())); |
| 216 | const problems = [ |
| 217 | ...required |
| 218 | .filter(name => !copies.some(copy => copy.name === name)) |
| 219 | .map(name => `${name} is not installed`), |
| 220 | ...copies |
| 221 | .filter(copy => copy.version !== versions.get(copy.name)) |
| 222 | .map(copy => `${copy.directory} is ${copy.version}, not ${versions.get(copy.name)}`), |
| 223 | ]; |
| 224 | yield* ensure( |
| 225 | problems.length === 0, |
| 226 | `The docs did not install the candidate packages:\n${problems.join('\n')}` |
| 227 | ); |
| 228 | }); |
| 229 | |
| 230 | const bun = (docs: string, args: ReadonlyArray<string>, env: Record<string, string> = {}) => |
| 231 | run('bun', args, { cwd: docs, env: { ...withoutReleaseTokens, ...env }, output: 'inherit' }); |
| 232 | |
| 233 | /** |
| 234 | * Installs the docs dependencies for a stable release PR. Without a TypeScript release, the |
| 235 | * committed lock is installed from npm as is. Otherwise the planned packages come from the |
| 236 | * candidate through a job-local registry, every other package through its npm proxy, and every |
| 237 | * installed copy of a planned package must be its candidate version. |
| 238 | */ |
| 239 | export const installCandidateDocs = (docs: string, batch: ReleaseBatch, candidates: string) => |
| 240 | Effect.scoped( |
| 241 | Effect.gen(function* () { |
| 242 | const fs = yield* FileSystem.FileSystem; |
| 243 | const path = yield* Path.Path; |
| 244 | const npm = batch.plan.releases.filter(release => release.language === 'typescript'); |
| 245 | if (npm.length === 0) return yield* bun(docs, ['install', '--frozen-lockfile']); |
| 246 | const packages = yield* loadCandidatePackages(candidates, npm); |
| 247 | const registry = yield* serveCandidateRegistry(packages); |
| 248 | const direct = yield* pinCandidateVersions(path.join(docs, 'package.json'), npm); |
| 249 | // A fresh cache, so no planned package is reused from an earlier install. |
| 250 | const cache = yield* fs.makeTempDirectoryScoped({ prefix: 'sdk-candidate-docs-cache-' }); |
| 251 | yield* bun(docs, ['install'], { |
| 252 | NPM_CONFIG_REGISTRY: registry, |
| 253 | BUN_INSTALL_CACHE_DIR: cache, |
| 254 | }); |
| 255 | yield* assertCandidateVersions(path.join(docs, 'node_modules'), npm, direct); |
| 256 | }) |
| 257 | ); |
| 258 | |
| 259 | /** `candidate-registry.ts <candidate-dir>` installs the docs against a stable release's candidate. */ |
| 260 | const main = Effect.gen(function* () { |
| 261 | const path = yield* Path.Path; |
| 262 | const [candidates] = yield* Schema.decodeUnknownEffect(Schema.Tuple([Schema.String]))( |
| 263 | yield* (yield* Stdio.Stdio).args |
| 264 | ); |
| 265 | const batch = yield* readJson(ReleaseBatch, BATCH_PATH); |
| 266 | yield* installCandidateDocs(path.resolve('docs'), batch, path.resolve(candidates)); |
| 267 | }); |
| 268 | |
| 269 | if (import.meta.main) runMain(main); |