Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | import { Config, Effect, FileSystem, Option, Path, Schema, Stdio } from 'effect'; |
| 2 | import { getPackages } from '@manypkg/get-packages'; |
| 3 | import { isReleaseAppCommit } from './approval'; |
| 4 | import { |
| 5 | ActionArtifact, |
| 6 | ApprovedCandidates, |
| 7 | CANDIDATE_MANIFEST, |
| 8 | type ApprovedCandidate, |
| 9 | CandidateManifest, |
| 10 | candidateArtifactName, |
| 11 | downloadCandidate, |
| 12 | normalizeArtifactDigest, |
| 13 | originalCandidates, |
| 14 | readCandidateFiles, |
| 15 | reuseCandidate, |
| 16 | verifyCandidateFiles, |
| 17 | type CandidateArtifact, |
| 18 | } from './artifacts'; |
| 19 | import { |
| 20 | GitHub, |
| 21 | PositiveIntFromString, |
| 22 | REPOSITORY_PATH, |
| 23 | ReleaseRun, |
| 24 | SDK_RELEASE_REPOSITORY, |
| 25 | Sha, |
| 26 | jobs, |
| 27 | request, |
| 28 | } from './github'; |
| 29 | import { Language, REGISTRY_VERSION } from './lanes'; |
| 30 | import { applySdkReleasePlan, type SdkRelease } from './plan'; |
| 31 | import { discoverPythonPackages } from './python-metadata'; |
| 32 | import { |
| 33 | appendFile, |
| 34 | ensure, |
| 35 | fail, |
| 36 | formatJson, |
| 37 | fromPromise, |
| 38 | inDirectory, |
| 39 | readJson, |
| 40 | runMain, |
| 41 | sha256Hex, |
| 42 | type ReleaseServices, |
| 43 | } from './runtime'; |
| 44 | import { BATCH_PATH, ReleaseBatch, batchLanguages, channelOf, releaseTag } from './source'; |
| 45 | |
| 46 | export const PackedManifest = Schema.StructWithRest( |
| 47 | Schema.Struct({ name: Schema.String, version: Schema.String }), |
| 48 | [Schema.Record(Schema.String, Schema.Unknown)] |
| 49 | ); |
| 50 | |
| 51 | export const tarballName = (name: string, version: string) => |
| 52 | `${name.replace(/^@/, '').replaceAll('/', '-')}-${version}.tgz`; |
| 53 | |
| 54 | /** |
| 55 | * Builds the SDK packages and packs the selected ones into `directory`. Provider compatibility is |
| 56 | * checked by ts.test.yml; this job runs no registry installs before the files are hashed. |
| 57 | */ |
| 58 | export const buildTypescript = (cwd: string, directory: string, names: ReadonlySet<string>) => |
| 59 | Effect.gen(function* () { |
| 60 | const path = yield* Path.Path; |
| 61 | const exec = inDirectory(cwd); |
| 62 | const { packages } = yield* fromPromise(() => getPackages(cwd)); |
| 63 | const selected = packages.filter(pkg => names.has(pkg.packageJson.name)); |
| 64 | yield* exec('pnpm', [ |
| 65 | 'exec', |
| 66 | 'turbo', |
| 67 | 'build', |
| 68 | ...selected.map(pkg => `--filter=${pkg.packageJson.name}`), |
| 69 | ]); |
| 70 | for (const pkg of selected) { |
| 71 | yield* exec('pnpm', [ |
| 72 | '--dir', |
| 73 | pkg.dir, |
| 74 | 'pack', |
| 75 | '--pack-destination', |
| 76 | directory, |
| 77 | '--config.ignore-scripts=true', |
| 78 | ]); |
| 79 | const name = tarballName(pkg.packageJson.name, pkg.packageJson.version); |
| 80 | const packed = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(PackedManifest))( |
| 81 | yield* exec('tar', ['-xOf', path.join(directory, name), 'package/package.json']) |
| 82 | ); |
| 83 | yield* ensure( |
| 84 | packed.name === pkg.packageJson.name && packed.version === pkg.packageJson.version, |
| 85 | 'Packed npm metadata differs from the approved source' |
| 86 | ); |
| 87 | } |
| 88 | }); |
| 89 | |
| 90 | const PYTHON_DISTRIBUTION = /\.whl$|\.tar\.gz$/; |
| 91 | |
| 92 | /** Builds the synchronized Python train with the locked release toolchain and keeps its distributions. */ |
| 93 | export const buildPython = (cwd: string, directory: string) => |
| 94 | Effect.gen(function* () { |
| 95 | const fs = yield* FileSystem.FileSystem; |
| 96 | const path = yield* Path.Path; |
| 97 | const environment = { UV_PROJECT_ENVIRONMENT: path.resolve(cwd, 'python/.venv') }; |
| 98 | const exec = inDirectory(cwd, environment); |
| 99 | yield* exec('uv', ['sync', '--locked', '--only-group', 'release']); |
| 100 | yield* inDirectory(path.join(cwd, 'python'), environment)('make', [ |
| 101 | 'build', |
| 102 | 'BUILD_ARGS=--no-isolation', |
| 103 | ]); |
| 104 | yield* exec('uv', [ |
| 105 | 'run', |
| 106 | '--locked', |
| 107 | '--only-group', |
| 108 | 'release', |
| 109 | 'python', |
| 110 | 'scripts/sdk-release/validate-python-candidates.py', |
| 111 | ]); |
| 112 | const dist = path.resolve(cwd, 'python/dist'); |
| 113 | for (const name of (yield* fs.readDirectory(dist)).filter(name => |
| 114 | PYTHON_DISTRIBUTION.test(name) |
| 115 | )) |
| 116 | yield* fs.copyFile(path.join(dist, name), path.join(directory, name)); |
| 117 | }); |
| 118 | |
| 119 | /** Lane-specific source checks and builds for a fresh candidate. */ |
| 120 | const LANE_BUILDS: Record< |
| 121 | Language, |
| 122 | { |
| 123 | readonly verifyTrain: ( |
| 124 | cwd: string, |
| 125 | selected: ReadonlyArray<SdkRelease> |
| 126 | ) => Effect.Effect<void, unknown, ReleaseServices>; |
| 127 | readonly build: ( |
| 128 | cwd: string, |
| 129 | directory: string, |
| 130 | selected: ReadonlyArray<SdkRelease> |
| 131 | ) => Effect.Effect<void, unknown, ReleaseServices>; |
| 132 | } |
| 133 | > = { |
| 134 | typescript: { |
| 135 | verifyTrain: () => Effect.void, |
| 136 | build: (cwd, directory, selected) => |
| 137 | buildTypescript(cwd, directory, new Set(selected.map(release => release.name))), |
| 138 | }, |
| 139 | python: { |
| 140 | verifyTrain: (cwd, selected) => |
| 141 | Effect.gen(function* () { |
| 142 | const [release] = selected; |
| 143 | const python = yield* discoverPythonPackages(cwd); |
| 144 | yield* ensure( |
| 145 | python.every(pkg => pkg.version === REGISTRY_VERSION.python(release.newVersion)), |
| 146 | 'Python candidate train differs from the approved target' |
| 147 | ); |
| 148 | }), |
| 149 | build: (cwd, directory) => buildPython(cwd, directory), |
| 150 | }, |
| 151 | }; |
| 152 | |
| 153 | const BuildEnvironment = Config.all({ |
| 154 | runId: Config.schema(PositiveIntFromString, 'GITHUB_RUN_ID'), |
| 155 | /** The release PR head; on pull_request, GITHUB_SHA is the merge commit instead. */ |
| 156 | sha: Config.schema(Sha, 'SDK_SOURCE_SHA'), |
| 157 | ref: Config.String('GITHUB_REF'), |
| 158 | controllerToken: Config.option(Config.String('SDK_RELEASE_TOKEN')), |
| 159 | }); |
| 160 | type BuildEnvironment = Config.Success<typeof BuildEnvironment>; |
| 161 | |
| 162 | /** Release-PR runs build candidates; source-tag runs publish the approved release-PR bytes. */ |
| 163 | const isSourceTag = (ref: string) => ref.startsWith('refs/tags/sdk-release/'); |
| 164 | |
| 165 | /** The pull_request-only workflow that builds release-PR candidates. */ |
| 166 | const CANDIDATES_WORKFLOW = 'sdk.release.candidates.yml'; |
| 167 | const CANDIDATES_WORKFLOW_PATH = `.github/workflows/${CANDIDATES_WORKFLOW}`; |
| 168 | |
| 169 | const WorkflowRuns = Schema.Struct({ workflow_runs: Schema.Array(ReleaseRun) }); |
| 170 | |
| 171 | /** GitHub reports step and artifact times to the second, so an artifact can land on either boundary second. */ |
| 172 | const TIMESTAMP_SLACK_MS = 1_000; |
| 173 | |
| 174 | /** The release-PR build step that uploads the candidate artifact. */ |
| 175 | const UPLOAD_STEP = 'Retain candidate bytes for publication'; |
| 176 | |
| 177 | /** |
| 178 | * Requires `artifact` to have been created by the upload step of the newest successful `jobName` |
| 179 | * job of `runId`. Every other job and step in that run holds its runtime token and could replace |
| 180 | * the artifact. Jobs from every attempt are read because a partial re-run leaves earlier jobs out |
| 181 | * of the latest attempt. |
| 182 | */ |
| 183 | const requireCreatedDuring = ( |
| 184 | runId: number, |
| 185 | jobName: string, |
| 186 | artifact: CandidateArtifact, |
| 187 | where: string |
| 188 | ) => |
| 189 | Effect.gen(function* () { |
| 190 | const [step] = (yield* jobs(String(runId), 'all')) |
| 191 | .filter(job => job.name === jobName && job.conclusion === 'success') |
| 192 | .flatMap(job => job.steps) |
| 193 | .flatMap(step => |
| 194 | step.name === UPLOAD_STEP && |
| 195 | step.conclusion === 'success' && |
| 196 | step.started_at && |
| 197 | step.completed_at |
| 198 | ? [{ startedAt: step.started_at, completedAt: step.completed_at }] |
| 199 | : [] |
| 200 | ) |
| 201 | .toSorted((a, b) => Date.parse(b.completedAt) - Date.parse(a.completedAt)); |
| 202 | if (!step) |
| 203 | return yield* fail( |
| 204 | `The ${jobName} job did not upload a candidate in ${where}; add changesets and request a new release` |
| 205 | ); |
| 206 | const created = Date.parse(artifact.created_at); |
| 207 | yield* ensure( |
| 208 | Date.parse(step.startedAt) - TIMESTAMP_SLACK_MS <= created && |
| 209 | created <= Date.parse(step.completedAt) + TIMESTAMP_SLACK_MS, |
| 210 | `${artifact.name} was created at ${artifact.created_at}, outside its upload step, which ran from ${step.startedAt} to ${step.completedAt}; add changesets and request a new release` |
| 211 | ); |
| 212 | }); |
| 213 | |
| 214 | /** |
| 215 | * Copies the candidate a release-PR run built and checked on this exact head into `directory`. |
| 216 | * Every successful release-PR run on the head passed the candidate checks the merge requires; |
| 217 | * the newest one is used, and its artifact must come from that run's build job. |
| 218 | */ |
| 219 | export const adoptApprovedCandidate = (sha: string, language: Language, directory: string) => |
| 220 | Effect.gen(function* () { |
| 221 | const { workflow_runs } = yield* request( |
| 222 | 'GET', |
| 223 | `${REPOSITORY_PATH}/actions/workflows/${CANDIDATES_WORKFLOW}/runs?event=pull_request&head_sha=${sha}&status=success&per_page=100`, |
| 224 | WorkflowRuns |
| 225 | ); |
| 226 | const [run] = workflow_runs |
| 227 | .filter( |
| 228 | candidate => |
| 229 | candidate.path === CANDIDATES_WORKFLOW_PATH && |
| 230 | candidate.repository.full_name === SDK_RELEASE_REPOSITORY && |
| 231 | candidate.event === 'pull_request' && |
| 232 | candidate.head_sha === sha && |
| 233 | candidate.conclusion === 'success' |
| 234 | ) |
| 235 | .toSorted((a, b) => b.id - a.id); |
| 236 | if (!run) |
| 237 | return yield* fail( |
| 238 | `No successful release-PR run built the approved ${language} candidate; add changesets and request a new release` |
| 239 | ); |
| 240 | const artifact = yield* reuseCandidate( |
| 241 | yield* originalCandidates(run.id), |
| 242 | candidateArtifactName(language), |
| 243 | run.id, |
| 244 | sha |
| 245 | ); |
| 246 | if (Option.isNone(artifact)) |
| 247 | return yield* fail( |
| 248 | `Approved ${language} candidate was deleted from ${run.html_url}; add changesets and request a new release` |
| 249 | ); |
| 250 | yield* requireCreatedDuring( |
| 251 | run.id, |
| 252 | `SDK candidate (${language})`, |
| 253 | artifact.value, |
| 254 | run.html_url |
| 255 | ); |
| 256 | yield* downloadCandidate(artifact.value, directory); |
| 257 | const manifestPath = `${directory}/${CANDIDATE_MANIFEST}`; |
| 258 | const manifest = yield* readJson(CandidateManifest, manifestPath); |
| 259 | yield* ensure( |
| 260 | manifest.sourceSha === sha && |
| 261 | manifest.producerRun === run.id && |
| 262 | manifest.language === language, |
| 263 | 'Approved candidate manifest producer/source mismatch' |
| 264 | ); |
| 265 | yield* verifyCandidateFiles(yield* readCandidateFiles(directory), manifest.files); |
| 266 | yield* (yield* FileSystem.FileSystem).remove(manifestPath); |
| 267 | return { |
| 268 | run: run.id, |
| 269 | artifactId: artifact.value.id, |
| 270 | digest: artifact.value.digest, |
| 271 | } satisfies ApprovedCandidate; |
| 272 | }); |
| 273 | |
| 274 | /** |
| 275 | * Makes the checkout carry the versions the candidate publishes. A stable release PR already |
| 276 | * commits them; a beta PR commits only its plan, so its frozen plan is applied here, in the |
| 277 | * credential-free build and smoke jobs, and never reaches `next`. |
| 278 | */ |
| 279 | export const prepareCandidateSource = (cwd: string, batch: ReleaseBatch) => |
| 280 | channelOf(batch).commitsVersions ? Effect.void : applySdkReleasePlan(cwd, batch.plan); |
| 281 | |
| 282 | /** Builds a release-PR candidate. */ |
| 283 | const buildCandidate = ( |
| 284 | env: BuildEnvironment, |
| 285 | language: Language, |
| 286 | selected: ReadonlyArray<SdkRelease> |
| 287 | ) => |
| 288 | Effect.gen(function* () { |
| 289 | const fs = yield* FileSystem.FileSystem; |
| 290 | const path = yield* Path.Path; |
| 291 | const cwd = path.resolve('.'); |
| 292 | const directory = path.resolve('sdk-candidate', language); |
| 293 | yield* fs.makeDirectory(directory, { recursive: true }); |
| 294 | const lane = LANE_BUILDS[language]; |
| 295 | yield* lane.verifyTrain(cwd, selected); |
| 296 | const { packages } = yield* fromPromise(() => getPackages(cwd)); |
| 297 | for (const release of selected) { |
| 298 | const pkg = packages.find(candidate => candidate.packageJson.name === release.name); |
| 299 | yield* ensure( |
| 300 | pkg?.packageJson.version === release.newVersion, |
| 301 | `Candidate version differs from approved target: ${release.name}` |
| 302 | ); |
| 303 | } |
| 304 | yield* lane.build(cwd, directory, selected); |
| 305 | const files = yield* Effect.forEach(yield* fs.readDirectory(directory), name => |
| 306 | Effect.gen(function* () { |
| 307 | const bytes = yield* fs.readFile(`${directory}/${name}`); |
| 308 | return { name, sha256: yield* sha256Hex(bytes), size: bytes.length }; |
| 309 | }) |
| 310 | ); |
| 311 | const manifest = yield* Schema.decodeUnknownEffect(CandidateManifest)({ |
| 312 | sourceSha: env.sha, |
| 313 | producerRun: env.runId, |
| 314 | language, |
| 315 | files, |
| 316 | }); |
| 317 | yield* fs.writeFileString(`${directory}/${CANDIDATE_MANIFEST}`, formatJson(manifest)); |
| 318 | }); |
| 319 | |
| 320 | const buildMode = (language: Language) => |
| 321 | Effect.gen(function* () { |
| 322 | const env = yield* BuildEnvironment; |
| 323 | yield* ensure( |
| 324 | !Option.exists(env.controllerToken, token => token !== ''), |
| 325 | 'Candidate jobs must not receive the controller credential' |
| 326 | ); |
| 327 | yield* ensure( |
| 328 | !isSourceTag(env.ref), |
| 329 | 'Source-tag runs publish approved candidates, never builds' |
| 330 | ); |
| 331 | const batch = yield* readJson(ReleaseBatch, BATCH_PATH); |
| 332 | const selected = batch.plan.releases.filter(release => release.language === language); |
| 333 | yield* ensure(selected.length > 0, `This release plans no ${language} packages`); |
| 334 | yield* prepareCandidateSource((yield* Path.Path).resolve('.'), batch); |
| 335 | yield* buildCandidate(env, language, selected); |
| 336 | }); |
| 337 | |
| 338 | const ReportEnvironment = Config.all({ |
| 339 | artifactId: Config.schema(PositiveIntFromString, 'ARTIFACT_ID'), |
| 340 | artifactDigest: Config.String('ARTIFACT_DIGEST'), |
| 341 | sourceSha: Config.schema(Sha, 'SOURCE_SHA'), |
| 342 | runId: Config.schema(PositiveIntFromString, 'GITHUB_RUN_ID'), |
| 343 | summaryPath: Config.String('GITHUB_STEP_SUMMARY'), |
| 344 | token: Config.schema(Schema.NonEmptyString, 'SDK_READ_TOKEN'), |
| 345 | }); |
| 346 | |
| 347 | /** Confirms the uploaded artifact is this run's candidate and records its identity. */ |
| 348 | const reportMode = (language: Language) => |
| 349 | Effect.gen(function* () { |
| 350 | const env = yield* ReportEnvironment; |
| 351 | const artifact = yield* request( |
| 352 | 'GET', |
| 353 | `${REPOSITORY_PATH}/actions/artifacts/${env.artifactId}`, |
| 354 | ActionArtifact |
| 355 | ).pipe(Effect.provide(GitHub.layer(env.token))); |
| 356 | yield* reuseCandidate([artifact], candidateArtifactName(language), env.runId, env.sourceSha); |
| 357 | yield* ensure( |
| 358 | artifact.digest === (yield* normalizeArtifactDigest(env.artifactDigest)), |
| 359 | 'Uploaded artifact digest differs' |
| 360 | ); |
| 361 | const manifest = yield* readJson( |
| 362 | CandidateManifest, |
| 363 | `sdk-candidate/${language}/${CANDIDATE_MANIFEST}` |
| 364 | ); |
| 365 | const files = manifest.files |
| 366 | .map(file => `- ${file.name}: ${file.sha256} (${file.size} bytes)`) |
| 367 | .join('\n'); |
| 368 | yield* appendFile( |
| 369 | env.summaryPath, |
| 370 | `${language} candidate ID=${artifact.id} digest=${artifact.digest} producer=${env.runId} source=${env.sourceSha} expires=${artifact.expires_at}\n\n${files}\n` |
| 371 | ); |
| 372 | }); |
| 373 | |
| 374 | const ApproveEnvironment = Config.all({ |
| 375 | sha: Config.schema(Sha, 'GITHUB_SHA'), |
| 376 | appSlug: Config.String('SDK_RELEASE_APP_SLUG'), |
| 377 | ref: Config.String('GITHUB_REF'), |
| 378 | outputPath: Config.String('GITHUB_OUTPUT'), |
| 379 | summaryPath: Config.String('GITHUB_STEP_SUMMARY'), |
| 380 | token: Config.schema(Schema.NonEmptyString, 'SDK_READ_TOKEN'), |
| 381 | controllerToken: Config.option(Config.String('SDK_RELEASE_TOKEN')), |
| 382 | }); |
| 383 | |
| 384 | /** |
| 385 | * Checks the approved release-PR candidate of every selected language once and pins its artifact |
| 386 | * ID and digest as a job output. Publish jobs download exactly that artifact, and a retried job |
| 387 | * reuses the output, so every attempt publishes the same bytes. |
| 388 | */ |
| 389 | const approveMode = Effect.gen(function* () { |
| 390 | const env = yield* ApproveEnvironment; |
| 391 | yield* ensure( |
| 392 | !Option.exists(env.controllerToken, token => token !== ''), |
| 393 | 'Candidate jobs must not receive the controller credential' |
| 394 | ); |
| 395 | const path = yield* Path.Path; |
| 396 | const batch = yield* readJson(ReleaseBatch, BATCH_PATH); |
| 397 | yield* ensure( |
| 398 | env.ref === `refs/tags/${releaseTag(batch.id)}`, |
| 399 | `Approved candidates are pinned only by the release tag of batch ${batch.id}` |
| 400 | ); |
| 401 | // Written first, so the publish status still reaches the request issue if adoption fails. Only |
| 402 | // a plan the release App committed may name that issue: the tag ruleset should stop anyone |
| 403 | // else from pushing a release tag, and this keeps a forged one from steering the bot. |
| 404 | const trusted = yield* isReleaseAppCommit(env.sha, batch, env.appSlug).pipe( |
| 405 | Effect.provide(GitHub.layer(env.token)) |
| 406 | ); |
| 407 | if (trusted) yield* appendFile(env.outputPath, `issue=${batch.issue}\n`); |
| 408 | const approved: Record<string, ApprovedCandidate> = {}; |
| 409 | yield* Effect.gen(function* () { |
| 410 | for (const language of batchLanguages(batch)) { |
| 411 | const candidate = yield* adoptApprovedCandidate( |
| 412 | env.sha, |
| 413 | language, |
| 414 | path.resolve('sdk-candidate', language) |
| 415 | ); |
| 416 | approved[language] = candidate; |
| 417 | yield* appendFile( |
| 418 | env.summaryPath, |
| 419 | `${language}: approved release-PR run ${candidate.run}, artifact ${candidate.artifactId} (${candidate.digest})\n` |
| 420 | ); |
| 421 | } |
| 422 | }).pipe(Effect.provide(GitHub.layer(env.token))); |
| 423 | const pinned = yield* Schema.decodeUnknownEffect(ApprovedCandidates)(approved); |
| 424 | yield* appendFile( |
| 425 | env.outputPath, |
| 426 | `approved=${JSON.stringify(pinned)}\ntypescript=${'typescript' in pinned}\npython=${'python' in pinned}\n` |
| 427 | ); |
| 428 | }); |
| 429 | |
| 430 | /** |
| 431 | * `candidates.ts <language>` builds a release-PR candidate, `candidates.ts report <language>` |
| 432 | * records its identity, and `candidates.ts approve` pins the approved candidates in a tag run. |
| 433 | */ |
| 434 | const Arguments = Schema.Union([ |
| 435 | Schema.Tuple([Schema.Literal('approve')]), |
| 436 | Schema.Tuple([Language]), |
| 437 | Schema.Tuple([Schema.Literal('report'), Language]), |
| 438 | ]); |
| 439 | |
| 440 | const main = Effect.gen(function* () { |
| 441 | const args = yield* Schema.decodeUnknownEffect(Arguments)(yield* (yield* Stdio.Stdio).args); |
| 442 | if (args[0] === 'approve') return yield* approveMode; |
| 443 | return yield* args.length === 1 ? buildMode(args[0]) : reportMode(args[1]); |
| 444 | }); |
| 445 | |
| 446 | if (import.meta.main) runMain(main); |