Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | import { |
| 2 | Config, |
| 3 | Console, |
| 4 | Context, |
| 5 | Data, |
| 6 | Duration, |
| 7 | Effect, |
| 8 | FileSystem, |
| 9 | Layer, |
| 10 | Option, |
| 11 | Path, |
| 12 | Schema, |
| 13 | Stdio, |
| 14 | } from 'effect'; |
| 15 | import { ChildProcessSpawner } from 'effect/process'; |
| 16 | import { FetchHttpClient, HttpClient } from 'effect/http'; |
| 17 | import semver from 'semver'; |
| 18 | import { |
| 19 | ActionArtifact, |
| 20 | ApprovedCandidates, |
| 21 | CANDIDATE_MANIFEST, |
| 22 | CandidateManifest, |
| 23 | candidateArtifactName, |
| 24 | downloadCandidate, |
| 25 | readCandidateFiles, |
| 26 | reuseCandidate, |
| 27 | verifyCandidateFiles, |
| 28 | type CandidateFile, |
| 29 | } from './artifacts'; |
| 30 | import { GitHub, REPOSITORY_PATH, Sha, request } from './github'; |
| 31 | import { CHANNELS, type Channel } from './channel'; |
| 32 | import { |
| 33 | NPM_REGISTRY, |
| 34 | REGISTRY_LANES, |
| 35 | REGISTRY_VERSION, |
| 36 | Language, |
| 37 | type Language as Lang, |
| 38 | } from './lanes'; |
| 39 | import { verifyNpmProvenance, verifyPypiProvenance } from './provenance'; |
| 40 | import { |
| 41 | REGISTRY_WAIT, |
| 42 | isPending, |
| 43 | isSuccess, |
| 44 | pollRegistry, |
| 45 | registryDeadline, |
| 46 | type RegistryDeadline, |
| 47 | } from './registry-wait'; |
| 48 | import { discoverPythonPackages } from './python-metadata'; |
| 49 | import { |
| 50 | CommandError, |
| 51 | ReleaseError, |
| 52 | appendFile, |
| 53 | ensure, |
| 54 | fail, |
| 55 | parseJson, |
| 56 | readJson, |
| 57 | run, |
| 58 | runMain, |
| 59 | sha256Hex, |
| 60 | withoutReleaseTokens, |
| 61 | } from './runtime'; |
| 62 | import { BATCH_PATH, ReleaseBatch, batchLanguages, releaseTag } from './source'; |
| 63 | import { |
| 64 | NPM_TRUSTED_PUBLISHING_RUNBOOK, |
| 65 | checkNpmLogin, |
| 66 | npmTrustedPublisherSettings, |
| 67 | } from './npm-oidc'; |
| 68 | |
| 69 | /** One reviewed distribution file: an npm tarball or a Python wheel/sdist. */ |
| 70 | export interface Distribution { |
| 71 | readonly language: Lang; |
| 72 | readonly name: string; |
| 73 | readonly version: string; |
| 74 | readonly file: string; |
| 75 | readonly bytes: Uint8Array; |
| 76 | readonly sha256: string; |
| 77 | } |
| 78 | |
| 79 | /** An npm dist-tag of the release's channel that should point at `version`. */ |
| 80 | export interface ChannelTag { |
| 81 | readonly name: string; |
| 82 | readonly version: string; |
| 83 | readonly tag: string; |
| 84 | } |
| 85 | |
| 86 | /** What the registries already serve, what they are missing, and which channel tags lag. */ |
| 87 | export interface RegistryState { |
| 88 | readonly existing: ReadonlyArray<Distribution>; |
| 89 | readonly missing: ReadonlyArray<Distribution>; |
| 90 | readonly tagsToRepair: ReadonlyArray<ChannelTag>; |
| 91 | } |
| 92 | |
| 93 | const Url = Schema.String.check( |
| 94 | Schema.makeFilter(value => (URL.canParse(value) ? undefined : 'Expected a URL')) |
| 95 | ); |
| 96 | |
| 97 | const registryGet = (url: string) => |
| 98 | Effect.gen(function* () { |
| 99 | const client = yield* HttpClient.HttpClient; |
| 100 | return yield* client.get(url); |
| 101 | }).pipe(Effect.provideService(FetchHttpClient.RequestInit, { redirect: 'error' })); |
| 102 | |
| 103 | /** A registry that does not yet serve something it should; propagation may still fix it. */ |
| 104 | class RegistryIncomplete extends Data.TaggedError('RegistryIncomplete')<{ |
| 105 | readonly message: string; |
| 106 | }> {} |
| 107 | /** Fails at once on an answer waiting cannot fix, and as still pending on any other failure. */ |
| 108 | const served = (status: number, pending: string, failed = pending) => |
| 109 | isSuccess(status) |
| 110 | ? Effect.void |
| 111 | : isPending(status) |
| 112 | ? Effect.fail(new RegistryIncomplete({ message: pending })) |
| 113 | : fail(failed); |
| 114 | |
| 115 | /** Registry metadata at `url`, or `None` when the registry answers 404. */ |
| 116 | const registryJson = <S extends Schema.Top>(url: string, schema: S) => |
| 117 | Effect.gen(function* () { |
| 118 | const response = yield* registryGet(url); |
| 119 | if (response.status === 404) return Option.none<S['Type']>(); |
| 120 | yield* served( |
| 121 | response.status, |
| 122 | `Registry lookup incomplete: ${url} (${response.status})`, |
| 123 | `Registry lookup failed: ${url} (${response.status})` |
| 124 | ); |
| 125 | return Option.some(yield* parseJson(schema, yield* response.text)); |
| 126 | }).pipe(Effect.timeout(REGISTRY_WAIT.requestTimeout)); |
| 127 | |
| 128 | const registryBytes = (url: string, failure: (status: number) => string) => |
| 129 | Effect.gen(function* () { |
| 130 | const response = yield* registryGet(url); |
| 131 | yield* served(response.status, failure(response.status)); |
| 132 | return new Uint8Array(yield* response.arrayBuffer); |
| 133 | }).pipe(Effect.timeout(REGISTRY_WAIT.requestTimeout)); |
| 134 | |
| 135 | const servedBy = (language: Lang, url: string) => |
| 136 | URL.canParse(url) && |
| 137 | new URL(url).protocol === 'https:' && |
| 138 | new URL(url).hostname === REGISTRY_LANES[language].downloadHost; |
| 139 | |
| 140 | /** A file the registry already serves must be byte for byte the reviewed candidate. */ |
| 141 | const verifyOriginalBytes = (url: string, file: Distribution) => |
| 142 | Effect.gen(function* () { |
| 143 | yield* ensure(servedBy(file.language, url), `Unexpected registry download host: ${file.file}`); |
| 144 | const bytes = yield* registryBytes( |
| 145 | url, |
| 146 | status => `Registry download incomplete: ${file.file} (${status})` |
| 147 | ); |
| 148 | yield* ensure( |
| 149 | bytes.length === file.bytes.length && (yield* sha256Hex(bytes)) === file.sha256, |
| 150 | `Registry content conflict: ${file.name}@${file.version}/${file.file} differs from the approved candidate. Published files are never replaced; release a new version.` |
| 151 | ); |
| 152 | }); |
| 153 | |
| 154 | const PypiVersion = Schema.Struct({ |
| 155 | urls: Schema.Array( |
| 156 | Schema.Struct({ |
| 157 | filename: Schema.String, |
| 158 | url: Url, |
| 159 | digests: Schema.Struct({ sha256: Schema.String }), |
| 160 | }) |
| 161 | ), |
| 162 | }); |
| 163 | const NpmVersion = Schema.Struct({ |
| 164 | name: Schema.String, |
| 165 | version: Schema.String, |
| 166 | dist: Schema.Struct({ tarball: Url }), |
| 167 | }); |
| 168 | const NpmTagged = Schema.Struct({ version: Schema.String }); |
| 169 | const PackageIdentity = Schema.Struct({ name: Schema.String, version: Schema.String }); |
| 170 | |
| 171 | const pythonDistributionNames = (name: string, version: string) => { |
| 172 | const prefix = `${name.replaceAll('-', '_')}-${version}`; |
| 173 | return [`${prefix}-py3-none-any.whl`, `${prefix}.tar.gz`]; |
| 174 | }; |
| 175 | |
| 176 | interface LoadedCandidate { |
| 177 | readonly cwd: string; |
| 178 | readonly batch: ReleaseBatch; |
| 179 | readonly directory: string; |
| 180 | readonly manifest: CandidateManifest; |
| 181 | readonly bytes: ReadonlyMap<string, Uint8Array>; |
| 182 | } |
| 183 | |
| 184 | const distributionOf = ( |
| 185 | language: Lang, |
| 186 | identity: { name: string; version: string }, |
| 187 | entry: CandidateFile, |
| 188 | bytes: ReadonlyMap<string, Uint8Array> |
| 189 | ): Distribution => ({ |
| 190 | language, |
| 191 | name: identity.name, |
| 192 | version: identity.version, |
| 193 | file: entry.name, |
| 194 | bytes: bytes.get(entry.name) ?? new Uint8Array(), |
| 195 | sha256: entry.sha256, |
| 196 | }); |
| 197 | |
| 198 | /** |
| 199 | * npm: one tarball per package version, and the channel's dist-tag moved forward to it. A beta |
| 200 | * only ever reads and moves its own tag, never `latest`. |
| 201 | */ |
| 202 | const npmAdapter = { |
| 203 | reconcileGroup: (group: ReadonlyArray<Distribution>, channel: Channel) => |
| 204 | Effect.gen(function* () { |
| 205 | yield* ensure(group.length === 1, 'Expected exactly one npm tarball per package'); |
| 206 | const [file] = group; |
| 207 | const metadata = yield* registryJson( |
| 208 | REGISTRY_LANES.typescript.versionUrl(file.name, file.version), |
| 209 | NpmVersion |
| 210 | ); |
| 211 | if (Option.isSome(metadata)) { |
| 212 | const { name, version, dist } = metadata.value; |
| 213 | yield* ensure( |
| 214 | name === file.name && version === file.version, |
| 215 | 'Registry npm identity conflict' |
| 216 | ); |
| 217 | yield* verifyOriginalBytes(dist.tarball, file); |
| 218 | } |
| 219 | const { npmTag } = CHANNELS[channel]; |
| 220 | const current = yield* registryJson( |
| 221 | `${NPM_REGISTRY}/${encodeURIComponent(file.name)}/${npmTag}`, |
| 222 | NpmTagged |
| 223 | ); |
| 224 | const tagged = Option.getOrUndefined(Option.map(current, value => value.version)); |
| 225 | yield* ensure( |
| 226 | tagged === undefined || semver.valid(tagged) !== null, |
| 227 | `Invalid npm ${npmTag} tag: ${file.name}` |
| 228 | ); |
| 229 | return { |
| 230 | existing: Option.isSome(metadata) ? [file] : [], |
| 231 | missing: Option.isSome(metadata) ? [] : [file], |
| 232 | tagsToRepair: |
| 233 | tagged === undefined || semver.lt(tagged, file.version) |
| 234 | ? [{ name: file.name, version: file.version, tag: npmTag }] |
| 235 | : [], |
| 236 | } satisfies RegistryState; |
| 237 | }), |
| 238 | candidateDistributions: (candidate: LoadedCandidate) => |
| 239 | Effect.gen(function* () { |
| 240 | const path = yield* Path.Path; |
| 241 | const releases = candidate.batch.plan.releases.filter( |
| 242 | release => release.language === 'typescript' |
| 243 | ); |
| 244 | const files: Distribution[] = []; |
| 245 | for (const entry of candidate.manifest.files) { |
| 246 | const manifest = yield* parseJson( |
| 247 | PackageIdentity, |
| 248 | yield* run( |
| 249 | 'tar', |
| 250 | ['-xOf', path.join(candidate.directory, entry.name), 'package/package.json'], |
| 251 | { cwd: candidate.cwd, env: withoutReleaseTokens } |
| 252 | ) |
| 253 | ); |
| 254 | yield* ensure( |
| 255 | releases.some( |
| 256 | release => release.name === manifest.name && release.newVersion === manifest.version |
| 257 | ), |
| 258 | 'Candidate contains an unplanned npm package' |
| 259 | ); |
| 260 | files.push(distributionOf('typescript', manifest, entry, candidate.bytes)); |
| 261 | } |
| 262 | yield* ensure( |
| 263 | candidate.manifest.files.length === releases.length, |
| 264 | 'Candidate npm package inventory differs' |
| 265 | ); |
| 266 | return files; |
| 267 | }), |
| 268 | }; |
| 269 | |
| 270 | /** PyPI: a wheel and sdist for every package of the synchronized Python train. */ |
| 271 | const pythonAdapter = { |
| 272 | reconcileGroup: (group: ReadonlyArray<Distribution>, _channel: Channel) => |
| 273 | Effect.gen(function* () { |
| 274 | const [first] = group; |
| 275 | const metadata = yield* registryJson( |
| 276 | REGISTRY_LANES.python.versionUrl(first.name, first.version), |
| 277 | PypiVersion |
| 278 | ); |
| 279 | const published = Option.match(metadata, { onNone: () => [], onSome: value => value.urls }); |
| 280 | yield* ensure( |
| 281 | new Set(published.map(file => file.filename)).size === published.length && |
| 282 | published.every(file => group.some(candidate => candidate.file === file.filename)), |
| 283 | `Unreviewed PyPI file for ${first.name}@${first.version}; release a new version` |
| 284 | ); |
| 285 | const existing: Distribution[] = []; |
| 286 | const missing: Distribution[] = []; |
| 287 | for (const file of group) { |
| 288 | const match = published.find(item => item.filename === file.file); |
| 289 | if (!match) { |
| 290 | missing.push(file); |
| 291 | continue; |
| 292 | } |
| 293 | yield* ensure( |
| 294 | match.digests.sha256 === file.sha256, |
| 295 | `Registry content conflict: ${file.file}` |
| 296 | ); |
| 297 | yield* verifyOriginalBytes(match.url, file); |
| 298 | existing.push(file); |
| 299 | } |
| 300 | return { existing, missing, tagsToRepair: [] } satisfies RegistryState; |
| 301 | }), |
| 302 | candidateDistributions: (candidate: LoadedCandidate) => |
| 303 | Effect.gen(function* () { |
| 304 | const release = candidate.batch.plan.releases.find(entry => entry.language === 'python'); |
| 305 | const version = REGISTRY_VERSION.python(release?.newVersion ?? ''); |
| 306 | const files: Distribution[] = []; |
| 307 | for (const pkg of yield* discoverPythonPackages(candidate.cwd)) |
| 308 | for (const name of pythonDistributionNames(pkg.name, version)) { |
| 309 | const entry = candidate.manifest.files.find(file => file.name === name); |
| 310 | if (!entry) return yield* fail(`Original Python distribution missing: ${name}`); |
| 311 | files.push(distributionOf('python', { name: pkg.name, version }, entry, candidate.bytes)); |
| 312 | } |
| 313 | yield* ensure( |
| 314 | candidate.manifest.files.length === files.length, |
| 315 | 'Candidate Python file inventory differs' |
| 316 | ); |
| 317 | return files; |
| 318 | }), |
| 319 | }; |
| 320 | |
| 321 | const REGISTRY_ADAPTERS = { typescript: npmAdapter, python: pythonAdapter } as const; |
| 322 | |
| 323 | /** Splits reviewed files into those the registry already serves and those still missing. */ |
| 324 | export const reconcileRegistry = ( |
| 325 | files: ReadonlyArray<Distribution>, |
| 326 | channel: Channel = 'stable' |
| 327 | ) => |
| 328 | Effect.gen(function* () { |
| 329 | const groups = Map.groupBy(files, file => `${file.language}:${file.name}@${file.version}`); |
| 330 | const states: ReadonlyArray<RegistryState> = yield* Effect.forEach(groups.values(), group => |
| 331 | REGISTRY_ADAPTERS[group[0].language].reconcileGroup(group, channel) |
| 332 | ); |
| 333 | return { |
| 334 | existing: states.flatMap(state => state.existing), |
| 335 | missing: states.flatMap(state => state.missing), |
| 336 | tagsToRepair: states.flatMap(state => state.tagsToRepair), |
| 337 | } satisfies RegistryState; |
| 338 | }); |
| 339 | |
| 340 | const incompleteInventory = (state: RegistryState, channelTags: boolean) => |
| 341 | [ |
| 342 | ...state.missing.map(file => `${file.name}@${file.version}/${file.file}`), |
| 343 | ...(channelTags ? state.tagsToRepair : []).map( |
| 344 | tag => `${tag.name}@${tag.version} ${tag.tag} tag` |
| 345 | ), |
| 346 | ].join(', '); |
| 347 | |
| 348 | /** |
| 349 | * Polls the registries until every file is visible, and every channel tag too unless |
| 350 | * `channelTags` is false (right after an upload, before the channel tag moves). npm serves a new |
| 351 | * version only after its publish-time scan, so the wait runs until `deadline` (the registry wait |
| 352 | * deadline from now, when omitted). Served bytes that differ from the candidate fail at once. |
| 353 | */ |
| 354 | export const waitForRegistry = ( |
| 355 | files: ReadonlyArray<Distribution>, |
| 356 | options: { |
| 357 | readonly channelTags?: boolean; |
| 358 | readonly channel?: Channel; |
| 359 | readonly deadline?: RegistryDeadline; |
| 360 | } = {} |
| 361 | ) => |
| 362 | pollRegistry( |
| 363 | reconcileRegistry(files, options.channel).pipe( |
| 364 | Effect.flatMap(state => { |
| 365 | const missing = incompleteInventory(state, options.channelTags ?? true); |
| 366 | return missing |
| 367 | ? Effect.fail(new RegistryIncomplete({ message: missing })) |
| 368 | : Effect.succeed(state); |
| 369 | }) |
| 370 | ), |
| 371 | error => error instanceof RegistryIncomplete, |
| 372 | options.deadline |
| 373 | ).pipe( |
| 374 | Effect.catchTag('RegistryIncomplete', error => |
| 375 | Effect.fail( |
| 376 | new RegistryIncomplete({ |
| 377 | message: `Registry verification incomplete when the registry wait (${REGISTRY_WAIT.deadline}) ended: ${error.message}`, |
| 378 | }) |
| 379 | ) |
| 380 | ) |
| 381 | ); |
| 382 | |
| 383 | /** |
| 384 | * Downloads the approved release-PR artifact pinned for `language` and checks its files |
| 385 | * against the manifest the release-PR build recorded. |
| 386 | */ |
| 387 | export const loadOriginalDistributions = ( |
| 388 | cwd: string, |
| 389 | batch: ReleaseBatch, |
| 390 | sha: string, |
| 391 | approved: ApprovedCandidates, |
| 392 | language: Lang |
| 393 | ) => |
| 394 | Effect.gen(function* () { |
| 395 | const fs = yield* FileSystem.FileSystem; |
| 396 | const path = yield* Path.Path; |
| 397 | const pinned = approved[language]; |
| 398 | if (pinned === undefined) |
| 399 | return yield* fail(`No approved ${language} candidate was pinned for this run`); |
| 400 | const listed = yield* request( |
| 401 | 'GET', |
| 402 | `${REPOSITORY_PATH}/actions/artifacts/${pinned.artifactId}`, |
| 403 | ActionArtifact |
| 404 | ); |
| 405 | const reused = yield* reuseCandidate( |
| 406 | [listed], |
| 407 | candidateArtifactName(language), |
| 408 | pinned.run, |
| 409 | sha |
| 410 | ); |
| 411 | if (Option.isNone(reused) || reused.value.digest !== pinned.digest) |
| 412 | return yield* fail( |
| 413 | `The approved ${language} candidate artifact ${pinned.artifactId} changed; release a new version` |
| 414 | ); |
| 415 | const directory = path.resolve(cwd, 'sdk-publication', language); |
| 416 | yield* fs.remove(directory, { recursive: true, force: true }); |
| 417 | yield* downloadCandidate(reused.value, directory); |
| 418 | const manifest = yield* readJson(CandidateManifest, path.join(directory, CANDIDATE_MANIFEST)); |
| 419 | yield* ensure( |
| 420 | manifest.sourceSha === sha && |
| 421 | manifest.producerRun === pinned.run && |
| 422 | manifest.language === language, |
| 423 | 'Original candidate producer/source conflict' |
| 424 | ); |
| 425 | const bytes = yield* readCandidateFiles(directory); |
| 426 | yield* verifyCandidateFiles(bytes, manifest.files); |
| 427 | return yield* REGISTRY_ADAPTERS[language].candidateDistributions({ |
| 428 | cwd, |
| 429 | batch, |
| 430 | directory, |
| 431 | manifest, |
| 432 | bytes, |
| 433 | }); |
| 434 | }); |
| 435 | |
| 436 | /** The npm CLI, which publishes through trusted publishing and moves dist-tags. */ |
| 437 | export class NpmCli extends Context.Service< |
| 438 | NpmCli, |
| 439 | { readonly run: (args: ReadonlyArray<string>) => Effect.Effect<string, unknown> } |
| 440 | >()('sdk-release/NpmCli') { |
| 441 | static readonly layer = (cwd: string) => |
| 442 | Layer.effect( |
| 443 | NpmCli, |
| 444 | Effect.gen(function* () { |
| 445 | const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; |
| 446 | return { |
| 447 | run: (args: ReadonlyArray<string>) => |
| 448 | run('npm', args, { cwd, env: withoutReleaseTokens }).pipe( |
| 449 | Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner) |
| 450 | ), |
| 451 | }; |
| 452 | }) |
| 453 | ); |
| 454 | } |
| 455 | |
| 456 | /** |
| 457 | * Runs `npm dist-tag add|rm` for `name`. npm refuses a trusted publisher without the dist-tag |
| 458 | * permission with a bare E403, so that refusal is reported with the setting that fixes it. |
| 459 | */ |
| 460 | export const npmDistTag = (args: readonly ['add' | 'rm', string, string], name: string) => |
| 461 | Effect.gen(function* () { |
| 462 | const cli = yield* NpmCli; |
| 463 | return yield* cli.run(['dist-tag', ...args]); |
| 464 | }).pipe( |
| 465 | Effect.mapError(error => |
| 466 | error instanceof CommandError && error.message.includes('cannot manage dist-tags') |
| 467 | ? new ReleaseError({ |
| 468 | message: `npm refused \`npm dist-tag ${args.join(' ')}\`: the trusted publisher of ${name} lacks the "Allow npm dist-tag" permission. A package maintainer must enable it at ${npmTrustedPublisherSettings(name)}; the npm CLI cannot. Then re-run this job. See ${NPM_TRUSTED_PUBLISHING_RUNBOOK}.\n${error.message}`, |
| 469 | }) |
| 470 | : error |
| 471 | ) |
| 472 | ); |
| 473 | |
| 474 | /** |
| 475 | * How npm 11.21.0 refuses to upload a version the registry already holds. npm holds a version |
| 476 | * from its upload on, before its publish-time scan ends and the registry serves it, so a re-run |
| 477 | * during the scan meets this refusal. |
| 478 | */ |
| 479 | const alreadyUploaded = (error: unknown, file: Distribution) => |
| 480 | error instanceof CommandError && |
| 481 | error.message.includes( |
| 482 | `You cannot publish over the previously published versions: ${file.version}.` |
| 483 | ); |
| 484 | |
| 485 | /** |
| 486 | * Checks the npm trusted-publishing login for every package, publishes under a release-only |
| 487 | * dist-tag, checks that each tarball's provenance names this release, then moves the channel's |
| 488 | * tag and drops the release-only tag. A failed check leaves the channel tag where it was. A |
| 489 | * version npm already holds is not uploaded again: the wait for the registry then compares the |
| 490 | * bytes npm serves with the reviewed tarball. |
| 491 | */ |
| 492 | export const publishNpm = (cwd: string, batch: ReleaseBatch, files: ReadonlyArray<Distribution>) => |
| 493 | Effect.gen(function* () { |
| 494 | const path = yield* Path.Path; |
| 495 | const channel = batch.plan.choices.channel; |
| 496 | const cli = yield* NpmCli; |
| 497 | const npm = cli.run; |
| 498 | yield* checkNpmLogin(files.map(file => file.name)); |
| 499 | for (const file of (yield* reconcileRegistry(files, channel)).missing) |
| 500 | yield* npm([ |
| 501 | 'publish', |
| 502 | path.resolve(cwd, 'sdk-publication/typescript', file.file), |
| 503 | '--ignore-scripts', |
| 504 | '--provenance', |
| 505 | '--access=public', |
| 506 | `--tag=sdk-release-${batch.id}`, |
| 507 | ]).pipe( |
| 508 | Effect.catchIf( |
| 509 | error => alreadyUploaded(error, file), |
| 510 | () => |
| 511 | Console.log( |
| 512 | `npm already holds ${file.name}@${file.version} but does not serve it yet, as during its publish-time scan; waiting for it instead of uploading again.` |
| 513 | ) |
| 514 | ) |
| 515 | ); |
| 516 | // Visibility and provenance share one deadline, which npm's publish-time scan fits in. |
| 517 | const deadline = yield* registryDeadline; |
| 518 | const visible = yield* waitForRegistry(files, { channelTags: false, channel, deadline }); |
| 519 | // Users only install a version once its channel tag moves, so it moves after the checks. |
| 520 | for (const file of files) yield* verifyNpmProvenance(file, batch.id, deadline); |
| 521 | for (const tag of visible.tagsToRepair) |
| 522 | yield* npmDistTag(['add', `${tag.name}@${tag.version}`, tag.tag], tag.name); |
| 523 | yield* waitForRegistry(files, { channel }); |
| 524 | // The release-only tag only served the upload; a re-run finds it already gone. |
| 525 | const releaseOnly = `sdk-release-${batch.id}`; |
| 526 | for (const file of files) { |
| 527 | const tags = yield* registryJson( |
| 528 | `${NPM_REGISTRY}/-/package/${encodeURIComponent(file.name)}/dist-tags`, |
| 529 | Schema.Record(Schema.String, Schema.String) |
| 530 | ); |
| 531 | if (Option.exists(tags, value => releaseOnly in value)) |
| 532 | yield* npmDistTag(['rm', file.name, releaseOnly], file.name); |
| 533 | } |
| 534 | }); |
| 535 | |
| 536 | /** The `name@version` specs of the published tarballs, for a fresh install. */ |
| 537 | export const npmSpecs = (files: ReadonlyArray<Distribution>) => |
| 538 | files.map(file => `${file.name}@${file.version}`).join(' '); |
| 539 | |
| 540 | /** |
| 541 | * Checks that every PyPI file is attested by this release, and returns the file URLs for the |
| 542 | * signature check that `pypi-attestations` runs next. |
| 543 | */ |
| 544 | export const verifyPypiFiles = (files: ReadonlyArray<Distribution>, deadline?: RegistryDeadline) => |
| 545 | Effect.gen(function* () { |
| 546 | const urls: string[] = []; |
| 547 | for (const file of files) { |
| 548 | yield* verifyPypiProvenance(file, deadline); |
| 549 | const metadata = yield* registryJson( |
| 550 | REGISTRY_LANES.python.versionUrl(file.name, file.version), |
| 551 | PypiVersion |
| 552 | ); |
| 553 | const served = Option.flatMap(metadata, ({ urls: served }) => |
| 554 | Option.fromNullishOr(served.find(entry => entry.filename === file.file)) |
| 555 | ); |
| 556 | if (Option.isNone(served)) return yield* fail(`PyPI does not serve ${file.file}`); |
| 557 | urls.push(served.value.url); |
| 558 | } |
| 559 | return urls; |
| 560 | }); |
| 561 | |
| 562 | /** Stages only the missing Python files for the trusted-publisher upload step. */ |
| 563 | const stageMissingPython = (cwd: string, files: ReadonlyArray<Distribution>, output: string) => |
| 564 | Effect.gen(function* () { |
| 565 | const fs = yield* FileSystem.FileSystem; |
| 566 | const path = yield* Path.Path; |
| 567 | const directory = path.resolve(cwd, 'sdk-publication/missing-python'); |
| 568 | yield* fs.remove(directory, { recursive: true, force: true }); |
| 569 | yield* fs.makeDirectory(directory, { recursive: true }); |
| 570 | const { missing } = yield* reconcileRegistry(files); |
| 571 | for (const file of missing) yield* fs.writeFile(path.join(directory, file.file), file.bytes); |
| 572 | yield* appendFile(output, `missing=${missing.length > 0}\n`); |
| 573 | }); |
| 574 | |
| 575 | const Environment = Config.all({ |
| 576 | token: Config.schema(Schema.NonEmptyString, 'SDK_READ_TOKEN'), |
| 577 | sha: Config.schema(Sha, 'GITHUB_SHA'), |
| 578 | ref: Config.String('GITHUB_REF'), |
| 579 | output: Config.String('GITHUB_OUTPUT'), |
| 580 | approved: Config.schema(Schema.fromJsonString(ApprovedCandidates), 'APPROVED_CANDIDATES'), |
| 581 | }); |
| 582 | |
| 583 | const Arguments = Schema.Union([ |
| 584 | Schema.Tuple([Schema.Literal('publish'), Language]), |
| 585 | Schema.Tuple([Schema.Literal('verify'), Schema.Literal('python')]), |
| 586 | ]); |
| 587 | |
| 588 | /** |
| 589 | * `registry.ts publish typescript` uploads to npm, moves the channel tag, checks provenance and |
| 590 | * outputs the specs for a signature check; `publish python` stages the missing PyPI files for the |
| 591 | * upload action; `verify python` waits until PyPI serves every file, checks each attestation's |
| 592 | * publisher and outputs the file URLs for the signature check. |
| 593 | */ |
| 594 | const main = Effect.gen(function* () { |
| 595 | const [mode, language] = yield* Schema.decodeUnknownEffect(Arguments)( |
| 596 | yield* (yield* Stdio.Stdio).args |
| 597 | ); |
| 598 | const env = yield* Environment; |
| 599 | const cwd = (yield* Path.Path).resolve('.'); |
| 600 | const batch = yield* readJson(ReleaseBatch, BATCH_PATH); |
| 601 | yield* ensure( |
| 602 | env.ref === `refs/tags/${releaseTag(batch.id)}`, |
| 603 | `Registry jobs require the release tag of batch ${batch.id}` |
| 604 | ); |
| 605 | yield* ensure(batchLanguages(batch).has(language), `This release plans no ${language} packages`); |
| 606 | const files = yield* loadOriginalDistributions(cwd, batch, env.sha, env.approved, language).pipe( |
| 607 | Effect.provide(GitHub.layer(env.token)) |
| 608 | ); |
| 609 | yield* Effect.gen(function* () { |
| 610 | if (mode === 'verify') { |
| 611 | const deadline = yield* registryDeadline; |
| 612 | yield* waitForRegistry(files, { channel: batch.plan.choices.channel, deadline }); |
| 613 | const urls = yield* verifyPypiFiles(files, deadline); |
| 614 | return yield* appendFile(env.output, `files=${urls.join(' ')}\n`); |
| 615 | } |
| 616 | if (language === 'typescript') { |
| 617 | yield* publishNpm(cwd, batch, files); |
| 618 | // The fresh-install step retries for as long as the registry wait. |
| 619 | return yield* appendFile( |
| 620 | env.output, |
| 621 | `specs=${npmSpecs(files)}\nwait_seconds=${Duration.toSeconds(REGISTRY_WAIT.deadline)}\n` |
| 622 | ); |
| 623 | } |
| 624 | yield* stageMissingPython(cwd, files, env.output); |
| 625 | }).pipe(Effect.provide(NpmCli.layer(cwd))); |
| 626 | }); |
| 627 | |
| 628 | if (import.meta.main) runMain(main); |