@devries-cameron20 / composio Mirror · read-only / commits / 35ab0feca
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 06:00 UTC

feat(release): tag approved releases and publish their candidates

On merge, the tag job checks the head is the release App's commit, approved by a writer other than the requester, with green candidate and docs checks, then tags it. The tag run pins the release-PR artifacts, uploads only the files npm and PyPI are missing, moves latest, and opens a docs PR that pins the released npm versions so twoslash checks the docs against them.

jkomyno committed Nov 5, 2026 at 17:48 UTC 35ab0feca76170ecec6d2d05b816cdb1bab15ba2
11 files changed +1987 -7
.github/workflows/sdk.release.yml
+216 -2
@@ -1,8 +1,14 @@
1 name: SDK release
2 run-name: SDK release (${{ github.event_name }}, ${{ github.actor }})
3
4 -# Stage 1 of the SDK release: a ComposioHQ member opens an `[SDK release]` issue or dispatches
5 -# this workflow, and the bot plans versions on protected next and opens one release PR.
4 +# One release, four stages, each started by its own event:
5 +# 1. request a ComposioHQ member opens an `[SDK release]` issue or dispatches this workflow;
6 +# the bot plans versions on protected next and opens one release PR.
7 +# 2. review sdk.release.candidates.yml builds and smoke-tests the files on the release PR;
8 +# a writer who did not request the release approves, and a person merges.
9 +# 3. tag on merge, the bot re-checks approval and checks, then tags the reviewed head.
10 +# 4. publish the tag run uploads the reviewed files to npm and PyPI, then opens a docs PR
11 +# that pins the released versions so twoslash checks against them.
12
13 on:
14 issues:
@@ -19,6 +25,11 @@ on:
25 type: choice
26 options: [auto, patch, minor, major, skip]
27 default: auto
28 + pull_request_target:
29 + branches: [next]
30 + types: [closed]
31 + push:
32 + tags: ['sdk-release/*']
33
34 permissions:
35 contents: read
@@ -137,3 +148,206 @@ jobs:
148 issue=$(jq -r .issue <<< "$BATCH")
149 gh issue comment "$issue" --repo "$GITHUB_REPOSITORY" \
150 --body "Preparing the release PR failed. Inspect $RUN_URL, then re-run its failed jobs or request the release again."
151 +
152 + # 3. tag -------------------------------------------------------------------------------------
153 +
154 + # Runs next as it was before this merge, so the merged release PR cannot change the gate.
155 + tag:
156 + if: >-
157 + github.repository == 'ComposioHQ/composio' && github.event_name == 'pull_request_target' &&
158 + github.event.pull_request.merged == true &&
159 + startsWith(github.event.pull_request.head.ref, 'sdk-release/batch-')
160 + runs-on: ubuntu-latest
161 + environment: sdk-release-control
162 + steps:
163 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
164 + with:
165 + ref: ${{ github.event.pull_request.base.sha }}
166 + persist-credentials: false
167 + - uses: ./.github/actions/setup-node-pnpm-bun
168 + - run: pnpm install --frozen-lockfile --ignore-scripts
169 + - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510
170 + id: app
171 + with:
172 + client-id: ${{ vars.SDK_RELEASE_APP_CLIENT_ID }}
173 + private-key: ${{ secrets.SDK_RELEASE_APP_PRIVATE_KEY }}
174 + owner: ComposioHQ
175 + repositories: composio
176 + - name: Check approval and candidate checks, then tag the reviewed head
177 + env:
178 + SDK_RELEASE_TOKEN: ${{ steps.app.outputs.token }}
179 + SDK_RELEASE_APP_SLUG: ${{ vars.SDK_RELEASE_APP_SLUG }}
180 + run: bun scripts/sdk-release/batch.ts tag
181 +
182 + # 4. publish ---------------------------------------------------------------------------------
183 +
184 + # Pins the release-PR artifacts once by ID and digest. "Re-run failed jobs" keeps this
185 + # output, so every attempt publishes the same reviewed bytes.
186 + candidates:
187 + if: github.repository == 'ComposioHQ/composio' && startsWith(github.ref, 'refs/tags/sdk-release/')
188 + runs-on: ubuntu-latest
189 + permissions:
190 + contents: read
191 + actions: read
192 + outputs:
193 + approved: ${{ steps.approve.outputs.approved }}
194 + typescript: ${{ steps.approve.outputs.typescript }}
195 + python: ${{ steps.approve.outputs.python }}
196 + steps:
197 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
198 + with:
199 + persist-credentials: false
200 + - uses: ./.github/actions/setup-node-pnpm-bun
201 + with:
202 + enable-caching: 'false'
203 + - run: pnpm install --frozen-lockfile --ignore-scripts
204 + - name: Pin the approved release-PR candidates
205 + id: approve
206 + env:
207 + SDK_READ_TOKEN: ${{ github.token }}
208 + run: bun scripts/sdk-release/candidates.ts approve
209 +
210 + publish-npm:
211 + needs: candidates
212 + if: needs.candidates.outputs.typescript == 'true'
213 + runs-on: ubuntu-latest
214 + environment: npm-release
215 + # One npm publisher at a time, so an older retry cannot move `latest` back over a newer release.
216 + concurrency:
217 + group: sdk-release-npm
218 + cancel-in-progress: false
219 + permissions:
220 + contents: read
221 + actions: read
222 + id-token: write
223 + steps:
224 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
225 + with:
226 + persist-credentials: false
227 + - uses: ./.github/actions/setup-node-pnpm-bun
228 + - run: pnpm install --frozen-lockfile --ignore-scripts
229 + - name: Pin npm with native dist-tag OIDC support
230 + run: npm install --global npm@11.21.0 --ignore-scripts --registry=https://registry.npmjs.org
231 + - name: Upload missing npm files, then move latest
232 + env:
233 + SDK_READ_TOKEN: ${{ github.token }}
234 + APPROVED_CANDIDATES: ${{ needs.candidates.outputs.approved }}
235 + run: bun scripts/sdk-release/registry.ts publish typescript
236 +
237 + publish-pypi:
238 + needs: candidates
239 + if: needs.candidates.outputs.python == 'true'
240 + runs-on: ubuntu-latest
241 + environment: pypi-release
242 + permissions:
243 + contents: read
244 + actions: read
245 + id-token: write
246 + steps:
247 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
248 + with:
249 + persist-credentials: false
250 + - uses: ./.github/actions/setup-node-pnpm-bun
251 + - run: pnpm install --frozen-lockfile --ignore-scripts
252 + - name: Stage the PyPI files that are still missing
253 + id: stage
254 + env:
255 + SDK_READ_TOKEN: ${{ github.token }}
256 + APPROVED_CANDIDATES: ${{ needs.candidates.outputs.approved }}
257 + run: bun scripts/sdk-release/registry.ts publish python
258 + - name: Upload missing PyPI files
259 + if: steps.stage.outputs.missing == 'true'
260 + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086
261 + with:
262 + packages-dir: sdk-publication/missing-python/
263 + - name: Wait until PyPI serves every file
264 + env:
265 + SDK_READ_TOKEN: ${{ github.token }}
266 + APPROVED_CANDIDATES: ${{ needs.candidates.outputs.approved }}
267 + run: bun scripts/sdk-release/registry.ts verify python
268 +
269 + docs:
270 + needs: [candidates, publish-npm, publish-pypi]
271 + if: >-
272 + !cancelled() && needs.candidates.result == 'success' &&
273 + contains(fromJSON('["success", "skipped"]'), needs.publish-npm.result) &&
274 + contains(fromJSON('["success", "skipped"]'), needs.publish-pypi.result)
275 + runs-on: ubuntu-latest
276 + permissions:
277 + contents: read
278 + issues: write
279 + steps:
280 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
281 + with:
282 + persist-credentials: false
283 + - uses: ./.github/actions/setup-node-pnpm-bun
284 + - run: pnpm install --frozen-lockfile --ignore-scripts
285 + - name: Report the published versions on the request issue
286 + id: released
287 + env:
288 + GITHUB_TOKEN: ${{ github.token }}
289 + run: bun scripts/sdk-release/report.ts released
290 + - if: steps.released.outputs.npm != ''
291 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
292 + with:
293 + ref: next
294 + path: next
295 + persist-credentials: false
296 + - name: Pin the docs to the released npm versions
297 + if: steps.released.outputs.npm != ''
298 + working-directory: next/docs
299 + env:
300 + RELEASED: ${{ steps.released.outputs.npm }}
301 + run: |
302 + specs=$(bun ../../scripts/sdk-release/report.ts docs-pins package.json)
303 + [ -n "$specs" ] || exit 0
304 + # The registry CDN can serve a stale package document right after publishing.
305 + for attempt in 1 2 3 4 5; do
306 + # shellcheck disable=SC2086 # one argument per package
307 + bun add --dev --exact --ignore-scripts $specs && exit 0
308 + echo "bun add failed (attempt $attempt); retrying in 30 seconds"
309 + sleep 30
310 + done
311 + exit 1
312 + - if: steps.released.outputs.npm != ''
313 + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510
314 + id: app
315 + with:
316 + client-id: ${{ vars.RELEASE_BOT_CLIENT_ID }}
317 + private-key: ${{ secrets.RELEASE_BOT_APP_PRIVATE_KEY }}
318 + - name: Open the docs pin PR
319 + if: steps.released.outputs.npm != ''
320 + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
321 + with:
322 + token: ${{ steps.app.outputs.token }}
323 + path: next
324 + base: next
325 + branch: docs/${{ github.ref_name }}
326 + add-paths: |
327 + docs/package.json
328 + docs/bun.lock
329 + commit-message: 'docs: pin the SDK packages released in ${{ github.ref_name }}'
330 + title: 'docs: pin the SDK packages released in ${{ github.ref_name }}'
331 + body: |
332 + Pins the docs to the npm versions published by ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}, so twoslash type-checks every snippet against the released types.
333 +
334 + Merge after the docs checks pass.
335 +
336 + report-failure:
337 + needs: [candidates, publish-npm, publish-pypi, docs]
338 + if: failure() && startsWith(github.ref, 'refs/tags/sdk-release/')
339 + runs-on: ubuntu-latest
340 + permissions:
341 + contents: read
342 + actions: read
343 + issues: write
344 + steps:
345 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
346 + with:
347 + persist-credentials: false
348 + - uses: ./.github/actions/setup-node-pnpm-bun
349 + - run: pnpm install --frozen-lockfile --ignore-scripts
350 + - name: Name the failed jobs and the re-run on the request issue
351 + env:
352 + GITHUB_TOKEN: ${{ github.token }}
353 + run: bun scripts/sdk-release/report.ts failed
scripts/sdk-release/approval.ts new
+167
@@ -0,0 +1,167 @@
1 +import { Effect, Option, Schema, Stream } from 'effect';
2 +import {
3 + CONTENT_CONCURRENCY,
4 + REPOSITORY_PATH,
5 + Sha,
6 + fileAt,
7 + paginate,
8 + paginateArray,
9 + request,
10 +} from './github';
11 +import { Pull, isReleasePull, releaseBranch } from './release-pr';
12 +import { ensure, fail } from './runtime';
13 +import { BATCH_PATH, ReleaseBatch, batchLanguages } from './source';
14 +
15 +const WRITE_PERMISSIONS = new Set(['write', 'maintain', 'admin']);
16 +const REVIEW_STATES = new Set(['APPROVED', 'CHANGES_REQUESTED', 'DISMISSED']);
17 +
18 +export interface Review {
19 + readonly user: string;
20 + readonly state: string;
21 + readonly commitId: string;
22 + readonly permission: string;
23 +}
24 +
25 +/**
26 + * A writer who is not excluded (the PR author and the requester) approved this exact head, and
27 + * no writer's latest review requests changes.
28 + */
29 +export function eligibleApproval(
30 + reviews: ReadonlyArray<Review>,
31 + excluded: ReadonlyArray<string>,
32 + head: string
33 +) {
34 + const latest = [...new Map(reviews.map(review => [review.user, review])).values()];
35 + const writers = latest.filter(review => WRITE_PERMISSIONS.has(review.permission));
36 + return (
37 + !writers.some(review => review.state === 'CHANGES_REQUESTED') &&
38 + writers.some(
39 + review =>
40 + !excluded.includes(review.user) && review.state === 'APPROVED' && review.commitId === head
41 + )
42 + );
43 +}
44 +
45 +const PullReview = Schema.Struct({
46 + user: Schema.Struct({ login: Schema.String }),
47 + state: Schema.String,
48 + commit_id: Sha,
49 +});
50 +const CollaboratorPermission = Schema.Struct({ permission: Schema.String });
51 +const CheckRuns = Schema.Struct({
52 + check_runs: Schema.Array(
53 + Schema.Struct({
54 + name: Schema.String,
55 + head_sha: Sha,
56 + app: Schema.NullOr(Schema.Struct({ slug: Schema.String })),
57 + conclusion: Schema.NullOr(Schema.String),
58 + })
59 + ),
60 +});
61 +const HeadCommit = Schema.Struct({
62 + author: Schema.NullOr(Schema.Struct({ login: Schema.String })),
63 + committer: Schema.NullOr(Schema.Struct({ login: Schema.String })),
64 + commit: Schema.Struct({ verification: Schema.Struct({ verified: Schema.Boolean }) }),
65 + parents: Schema.Array(Schema.Struct({ sha: Sha })),
66 +});
67 +
68 +/** Review decisions with each reviewer's repository permission, looked up once per login. */
69 +const reviewsWithPermissions = (number: number) =>
70 + Effect.gen(function* () {
71 + const decisions = (yield* Stream.runCollect(
72 + paginateArray(`${REPOSITORY_PATH}/pulls/${number}/reviews`, PullReview)
73 + )).filter(review => REVIEW_STATES.has(review.state));
74 + const logins = [...new Set(decisions.map(review => review.user.login))];
75 + const permissions = new Map(
76 + yield* Effect.forEach(
77 + logins,
78 + login =>
79 + Effect.map(
80 + request(
81 + 'GET',
82 + `${REPOSITORY_PATH}/collaborators/${encodeURIComponent(login)}/permission`,
83 + CollaboratorPermission
84 + ),
85 + result => [login, result.permission] as const
86 + ),
87 + { concurrency: CONTENT_CONCURRENCY }
88 + )
89 + );
90 + return decisions.map(
91 + review =>
92 + ({
93 + user: review.user.login,
94 + state: review.state,
95 + commitId: review.commit_id,
96 + permission: permissions.get(review.user.login) ?? '',
97 + }) satisfies Review
98 + );
99 + });
100 +
101 +/** A release PR into next and the plan committed on its head. */
102 +export const readReleasePull = (number: number) =>
103 + Effect.gen(function* () {
104 + const pull = yield* request('GET', `${REPOSITORY_PATH}/pulls/${number}`, Pull);
105 + yield* ensure(
106 + pull.base.ref === 'next' && isReleasePull(pull),
107 + 'Expected an SDK release PR into protected next'
108 + );
109 + const raw = yield* fileAt(BATCH_PATH, pull.head.sha);
110 + if (Option.isNone(raw)) return yield* fail('Missing committed release plan');
111 + const batch = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(ReleaseBatch))(raw.value);
112 + yield* ensure(
113 + pull.head.ref === releaseBranch(batch.id),
114 + 'Release branch and batch do not agree'
115 + );
116 + return { pull, batch };
117 + });
118 +
119 +/**
120 + * The checks a release PR must pass before its head is tagged:
121 + * - the head is the release App's own commit on the frozen source, so nobody added changes the
122 + * release-only diff check did not see;
123 + * - a writer other than the author and the requester approved that head;
124 + * - the candidate build, smoke and docs checks passed on it.
125 + */
126 +export const verifyReleaseGate = (pull: Pull, batch: ReleaseBatch, appSlug: string) =>
127 + Effect.gen(function* () {
128 + const head = yield* request('GET', `${REPOSITORY_PATH}/commits/${pull.head.sha}`, HeadCommit);
129 + yield* ensure(
130 + // A person can sign a commit that names the bot as author, but not one it committed.
131 + head.author?.login === `${appSlug}[bot]` &&
132 + head.committer?.login === `${appSlug}[bot]` &&
133 + head.commit.verification.verified &&
134 + head.parents.length === 1 &&
135 + head.parents[0].sha === batch.inputBase,
136 + 'The release PR head must be the release App commit on the planned source; request a new release instead of pushing to its branch'
137 + );
138 + const reviews = yield* reviewsWithPermissions(pull.number);
139 + yield* ensure(
140 + eligibleApproval(
141 + reviews,
142 + [pull.user?.login ?? '', batch.requestedBy].filter(Boolean),
143 + pull.head.sha
144 + ),
145 + 'Final release head lacks an approval from a writer other than the author and requester'
146 + );
147 + // A release head that touches every SDK runs more than one page of checks.
148 + const checks = yield* Stream.runCollect(
149 + paginate(
150 + `${REPOSITORY_PATH}/commits/${pull.head.sha}/check-runs`,
151 + CheckRuns,
152 + body => body.check_runs
153 + )
154 + );
155 + const passed = (name: string) =>
156 + checks.some(
157 + check =>
158 + check.name === name &&
159 + check.head_sha === pull.head.sha &&
160 + check.app?.slug === 'github-actions' &&
161 + check.conclusion === 'success'
162 + );
163 + for (const language of batchLanguages(batch))
164 + for (const check of [`SDK candidate (${language})`, `SDK candidate smoke (${language})`])
165 + yield* ensure(passed(check), `Missing green final-head check: ${check}`);
166 + yield* ensure(passed('typescript-check'), 'Missing green final-head docs check');
167 + });
scripts/sdk-release/artifacts.ts
+15
@@ -15,6 +15,21 @@ export const CandidateFile = Schema.Struct({
15 });
16 export type CandidateFile = typeof CandidateFile.Type;
17
18 +/** The release-PR candidate whose checked bytes a source-tag run publishes. */
19 +export const ApprovedCandidate = Schema.Struct({
20 + run: PositiveInt,
21 + artifactId: PositiveInt,
22 + digest: Digest,
23 +});
24 +export type ApprovedCandidate = typeof ApprovedCandidate.Type;
25 +
26 +/** The approved candidate of each selected language, pinned once per tag run. */
27 +export const ApprovedCandidates = Schema.Struct({
28 + typescript: Schema.optionalKey(ApprovedCandidate),
29 + python: Schema.optionalKey(ApprovedCandidate),
30 +});
31 +export type ApprovedCandidates = typeof ApprovedCandidates.Type;
32 +
33 export const CandidateManifest = Schema.Struct({
34 sourceSha: Sha,
35 producerRun: PositiveInt,
scripts/sdk-release/batch.ts
+80 -3
@@ -1,5 +1,6 @@
1 import { Cause, Config, Console, Effect, Option, Path, Schema, Stdio, Stream } from 'effect';
2 import { Base64 } from 'effect/encoding';
3 +import { readReleasePull, verifyReleaseGate } from './approval';
4 import {
5 CONTENT_CONCURRENCY,
6 CreatedSha,
@@ -8,6 +9,7 @@ import {
9 REPOSITORY_PATH,
10 RunId,
11 Sha,
12 + TagRef,
13 commentIssue,
14 fileAtOrNull,
15 paginateArray,
@@ -24,6 +26,7 @@ import {
26 ensure,
27 fail,
28 readJson,
29 + ReleaseError,
30 runMain,
31 sameJson,
32 type ReleaseServices,
@@ -253,6 +256,73 @@ export const openReleasePull = (cwd: string, planned: ReleaseBatch, value: unkno
256 return pull.html_url;
257 });
258
259 +// 3. tag ---------------------------------------------------------------------------------------
260 +
261 +/**
262 + * Explains a merged release that did not pass its gate on the PR and its request issue. Only a
263 + * gate verdict asks for a revert; a GitHub or transport error asks for a re-run instead.
264 + */
265 +const reportRefusedRelease = (pull: Pull, batch: ReleaseBatch, cause: Cause.Cause<unknown>) => {
266 + const body = (
267 + Cause.squash(cause) instanceof ReleaseError
268 + ? [
269 + `Release PR #${pull.number} merged, but it failed the release gate, so no publication tag was created and nothing was published.`,
270 + `Reason: ${causeMessage(cause)}`,
271 + 'Revert this release PR on next, then request the release again.',
272 + ]
273 + : [
274 + `Release PR #${pull.number} merged, but the release gate could not complete, so no publication tag was created and nothing was published.`,
275 + `Error: ${causeMessage(cause)}`,
276 + 'Use "Re-run failed jobs" on the tag job; creating the tag is safe to repeat.',
277 + ]
278 + ).join('\n\n');
279 + return Effect.gen(function* () {
280 + yield* commentIssue(pull.number, body);
281 + yield* commentIssue(batch.issue, body);
282 + }).pipe(
283 + Effect.catchCause(reportCause =>
284 + Console.error('Could not report the refused release:', causeMessage(reportCause))
285 + )
286 + );
287 +};
288 +
289 +/**
290 + * Creates the immutable publication tag on the reviewed head. GitHub records that head and the
291 + * squash commit separately, so a single-parent squash need not list the head as a parent.
292 + */
293 +export const authorizeSourceTag = (number: number, appSlug: string) =>
294 + Effect.gen(function* () {
295 + const { pull, batch } = yield* readReleasePull(number);
296 + if (!pull.merged || !pull.merge_commit_sha) return yield* fail('Release PR has not merged');
297 + yield* verifyReleaseGate(pull, batch, appSlug).pipe(
298 + Effect.tapCause(cause => reportRefusedRelease(pull, batch, cause))
299 + );
300 + const merged = yield* request(
301 + 'GET',
302 + `${REPOSITORY_PATH}/git/commits/${pull.merge_commit_sha}`,
303 + Commit
304 + );
305 + yield* ensure(
306 + merged.parents.some(parent => parent.sha === pull.head.sha) || merged.parents.length === 1,
307 + 'Unrecognized release merge topology'
308 + );
309 + const tag = `sdk-release/${batch.id}`;
310 + const existing = yield* requestOptional(`${REPOSITORY_PATH}/git/ref/tags/${tag}`, TagRef);
311 + yield* Option.match(existing, {
312 + onSome: ref =>
313 + ensure(
314 + ref.object.type === 'commit' && ref.object.sha === pull.head.sha,
315 + 'Publication source tag already points to another source'
316 + ),
317 + onNone: () =>
318 + request('POST', `${REPOSITORY_PATH}/git/refs`, CreatedRef, {
319 + ref: `refs/tags/${tag}`,
320 + sha: pull.head.sha,
321 + }),
322 + });
323 + return tag;
324 + });
325 +
326 // entry point ----------------------------------------------------------------------------------
327
328 const Environment = Config.all({
@@ -280,7 +350,7 @@ type Mode = (
350 envelope: typeof EventEnvelope.Type
351 ) => Effect.Effect<void, unknown, GitHub | ReleaseServices>;
352
283 -const MODES: Record<'plan' | 'release-pr', Mode> = {
353 +const MODES: Record<'plan' | 'release-pr' | 'tag', Mode> = {
354 plan: (env: Environment, event: unknown, envelope: typeof EventEnvelope.Type) =>
355 Effect.gen(function* () {
356 const cwd = (yield* Path.Path).resolve('.');
@@ -314,12 +384,19 @@ const MODES: Record<'plan' | 'release-pr', Mode> = {
384 url ? `Release PR: ${url}\n` : 'Another SDK release is in progress.\n'
385 );
386 }),
387 + tag: (env: Environment, _event: unknown, envelope: typeof EventEnvelope.Type) =>
388 + Effect.gen(function* () {
389 + if (!envelope.pull_request) return yield* fail('Expected merged release PR event');
390 + const appSlug = yield* Config.String('SDK_RELEASE_APP_SLUG');
391 + const tag = yield* authorizeSourceTag(envelope.pull_request.number, appSlug);
392 + yield* appendFile(env.summaryPath, `Approved publication source: ${tag}\n`);
393 + }),
394 };
395
319 -/** `batch.ts plan` or `batch.ts release-pr <patch>`. */
396 +/** `batch.ts plan`, `batch.ts release-pr <patch>`, or `batch.ts tag`. */
397 const main = Effect.gen(function* () {
398 const [argument] = yield* (yield* Stdio.Stdio).args;
322 - const mode = yield* Schema.decodeUnknownEffect(Schema.Literals(['plan', 'release-pr']))(
399 + const mode = yield* Schema.decodeUnknownEffect(Schema.Literals(['plan', 'release-pr', 'tag']))(
400 argument
401 );
402 const env = yield* Environment;
scripts/sdk-release/candidates.ts
+166 -2
@@ -2,17 +2,27 @@ import { Config, Effect, FileSystem, Option, Path, Schema, Stdio } from 'effect'
2 import { getPackages } from '@manypkg/get-packages';
3 import {
4 ActionArtifact,
5 + ApprovedCandidates,
6 CANDIDATE_MANIFEST,
7 + type ApprovedCandidate,
8 CandidateManifest,
9 candidateArtifactName,
10 + downloadCandidate,
11 normalizeArtifactDigest,
12 + originalCandidates,
13 + readCandidateFiles,
14 reuseCandidate,
15 + verifyCandidateFiles,
16 + type CandidateArtifact,
17 } from './artifacts';
18 import {
19 GitHub,
20 PositiveIntFromString,
21 REPOSITORY_PATH,
22 + ReleaseRun,
23 + SDK_RELEASE_REPOSITORY,
24 Sha,
25 + jobs,
26 request,
27 } from './github';
28 import { Language } from './lanes';
@@ -21,6 +31,7 @@ import { discoverPythonPackages } from './python-metadata';
31 import {
32 appendFile,
33 ensure,
34 + fail,
35 formatJson,
36 fromPromise,
37 inDirectory,
@@ -29,7 +40,7 @@ import {
40 sha256Hex,
41 type ReleaseServices,
42 } from './runtime';
32 -import { BATCH_PATH, ReleaseBatch } from './source';
43 +import { BATCH_PATH, ReleaseBatch, batchLanguages } from './source';
44
45 const PackedManifest = Schema.StructWithRest(
46 Schema.Struct({ name: Schema.String, version: Schema.String }),
@@ -151,6 +162,109 @@ type BuildEnvironment = Config.Success<typeof BuildEnvironment>;
162 /** Release-PR runs build candidates; source-tag runs publish the approved release-PR bytes. */
163 const isSourceTag = (ref: string) => ref.startsWith('refs/tags/sdk-release/');
164
165 +/** The pull_request-only workflow that builds release-PR candidates. */
166 +const CANDIDATES_WORKFLOW = 'sdk.release.candidates.yml';
167 +const CANDIDATES_WORKFLOW_PATH = `.github/workflows/${CANDIDATES_WORKFLOW}`;
168 +
169 +/** The release-PR build step that uploads the candidate artifact. */
170 +const UPLOAD_STEP = 'Retain candidate bytes for publication';
171 +
172 +/**
173 + * Requires `artifact` to have been created by the upload step of the newest successful `jobName`
174 + * job of `runId`. Every other job and step in that run holds its runtime token and could replace
175 + * the artifact. Jobs from every attempt are read because a partial re-run leaves earlier jobs out
176 + * of the latest attempt.
177 + */
178 +const requireCreatedDuring = (
179 + runId: number,
180 + jobName: string,
181 + artifact: CandidateArtifact,
182 + where: string
183 +) =>
184 + Effect.gen(function* () {
185 + const [step] = (yield* jobs(String(runId), 'all'))
186 + .filter(job => job.name === jobName && job.conclusion === 'success')
187 + .flatMap(job => job.steps)
188 + .flatMap(step =>
189 + step.name === UPLOAD_STEP &&
190 + step.conclusion === 'success' &&
191 + step.started_at &&
192 + step.completed_at
193 + ? [{ startedAt: step.started_at, completedAt: step.completed_at }]
194 + : []
195 + )
196 + .toSorted((a, b) => Date.parse(b.completedAt) - Date.parse(a.completedAt));
197 + if (!step)
198 + return yield* fail(
199 + `The ${jobName} job did not upload a candidate in ${where}; add changesets and request a new release`
200 + );
201 + const created = Date.parse(artifact.created_at);
202 + yield* ensure(
203 + Date.parse(step.startedAt) <= created && created <= Date.parse(step.completedAt),
204 + `${artifact.name} was created at ${artifact.created_at}, outside its upload step, which ran from ${step.startedAt} to ${step.completedAt}; add changesets and request a new release`
205 + );
206 + });
207 +
208 +/**
209 + * Copies the candidate a release-PR run built and checked on this exact head into `directory`.
210 + * Every successful release-PR run on the head passed the candidate checks the merge requires;
211 + * the newest one is used, and its artifact must come from that run's build job.
212 + */
213 +export const adoptApprovedCandidate = (sha: string, language: Language, directory: string) =>
214 + Effect.gen(function* () {
215 + const { workflow_runs } = yield* request(
216 + 'GET',
217 + `${REPOSITORY_PATH}/actions/workflows/${CANDIDATES_WORKFLOW}/runs?event=pull_request&head_sha=${sha}&status=success&per_page=100`,
218 + Schema.Struct({ workflow_runs: Schema.Array(ReleaseRun) })
219 + );
220 + const [run] = workflow_runs
221 + .filter(
222 + candidate =>
223 + candidate.path === CANDIDATES_WORKFLOW_PATH &&
224 + candidate.repository.full_name === SDK_RELEASE_REPOSITORY &&
225 + candidate.event === 'pull_request' &&
226 + candidate.head_sha === sha &&
227 + candidate.conclusion === 'success'
228 + )
229 + .toSorted((a, b) => b.id - a.id);
230 + if (!run)
231 + return yield* fail(
232 + `No successful release-PR run built the approved ${language} candidate; add changesets and request a new release`
233 + );
234 + const artifact = yield* reuseCandidate(
235 + yield* originalCandidates(run.id),
236 + candidateArtifactName(language),
237 + run.id,
238 + sha
239 + );
240 + if (Option.isNone(artifact))
241 + return yield* fail(
242 + `Approved ${language} candidate was deleted from ${run.html_url}; add changesets and request a new release`
243 + );
244 + yield* requireCreatedDuring(
245 + run.id,
246 + `SDK candidate (${language})`,
247 + artifact.value,
248 + run.html_url
249 + );
250 + yield* downloadCandidate(artifact.value, directory);
251 + const manifestPath = `${directory}/${CANDIDATE_MANIFEST}`;
252 + const manifest = yield* readJson(CandidateManifest, manifestPath);
253 + yield* ensure(
254 + manifest.sourceSha === sha &&
255 + manifest.producerRun === run.id &&
256 + manifest.language === language,
257 + 'Approved candidate manifest producer/source mismatch'
258 + );
259 + yield* verifyCandidateFiles(yield* readCandidateFiles(directory), manifest.files);
260 + yield* (yield* FileSystem.FileSystem).remove(manifestPath);
261 + return {
262 + run: run.id,
263 + artifactId: artifact.value.id,
264 + digest: artifact.value.digest,
265 + } satisfies ApprovedCandidate;
266 + });
267 +
268 /** Builds a release-PR candidate. */
269 const buildCandidate = (
270 env: BuildEnvironment,
@@ -243,14 +357,64 @@ const reportMode = (language: Language) =>
357 );
358 });
359
246 -/** `candidates.ts <language>` builds a release-PR candidate; `report <language>` records it. */
360 +const ApproveEnvironment = Config.all({
361 + sha: Config.schema(Sha, 'GITHUB_SHA'),
362 + ref: Config.String('GITHUB_REF'),
363 + outputPath: Config.String('GITHUB_OUTPUT'),
364 + summaryPath: Config.String('GITHUB_STEP_SUMMARY'),
365 + token: Config.schema(Schema.NonEmptyString, 'SDK_READ_TOKEN'),
366 + controllerToken: Config.option(Config.String('SDK_RELEASE_TOKEN')),
367 +});
368 +
369 +/**
370 + * Checks the approved release-PR candidate of every selected language once and pins its artifact
371 + * ID and digest as a job output. Publish jobs download exactly that artifact, and a retried job
372 + * reuses the output, so every attempt publishes the same bytes.
373 + */
374 +const approveMode = Effect.gen(function* () {
375 + const env = yield* ApproveEnvironment;
376 + yield* ensure(
377 + !Option.exists(env.controllerToken, token => token !== ''),
378 + 'Candidate jobs must not receive the controller credential'
379 + );
380 + yield* ensure(isSourceTag(env.ref), 'Approved candidates are pinned only by a source-tag run');
381 + const path = yield* Path.Path;
382 + const batch = yield* readJson(ReleaseBatch, BATCH_PATH);
383 + const approved: Record<string, ApprovedCandidate> = {};
384 + yield* Effect.gen(function* () {
385 + for (const language of batchLanguages(batch)) {
386 + const candidate = yield* adoptApprovedCandidate(
387 + env.sha,
388 + language,
389 + path.resolve('sdk-candidate', language)
390 + );
391 + approved[language] = candidate;
392 + yield* appendFile(
393 + env.summaryPath,
394 + `${language}: approved release-PR run ${candidate.run}, artifact ${candidate.artifactId} (${candidate.digest})\n`
395 + );
396 + }
397 + }).pipe(Effect.provide(GitHub.layer(env.token)));
398 + const pinned = yield* Schema.decodeUnknownEffect(ApprovedCandidates)(approved);
399 + yield* appendFile(
400 + env.outputPath,
401 + `approved=${JSON.stringify(pinned)}\ntypescript=${'typescript' in pinned}\npython=${'python' in pinned}\n`
402 + );
403 +});
404 +
405 +/**
406 + * `candidates.ts <language>` builds a release-PR candidate, `candidates.ts report <language>`
407 + * records its identity, and `candidates.ts approve` pins the approved candidates in a tag run.
408 + */
409 const Arguments = Schema.Union([
410 + Schema.Tuple([Schema.Literal('approve')]),
411 Schema.Tuple([Language]),
412 Schema.Tuple([Schema.Literal('report'), Language]),
413 ]);
414
415 const main = Effect.gen(function* () {
416 const args = yield* Schema.decodeUnknownEffect(Arguments)(yield* (yield* Stdio.Stdio).args);
417 + if (args[0] === 'approve') return yield* approveMode;
418 return yield* args.length === 1 ? buildMode(args[0]) : reportMode(args[1]);
419 });
420
scripts/sdk-release/registry.ts new
+456
@@ -0,0 +1,456 @@
1 +import { Config, Data, Effect, FileSystem, Option, Path, Schedule, Schema, Stdio } from 'effect';
2 +import type { Duration } from 'effect';
3 +import { FetchHttpClient, HttpClient } from 'effect/http';
4 +import semver from 'semver';
5 +import {
6 + ActionArtifact,
7 + ApprovedCandidates,
8 + CANDIDATE_MANIFEST,
9 + CandidateManifest,
10 + candidateArtifactName,
11 + downloadCandidate,
12 + readCandidateFiles,
13 + reuseCandidate,
14 + verifyCandidateFiles,
15 + type CandidateFile,
16 +} from './artifacts';
17 +import { GitHub, REPOSITORY_PATH, Sha, request } from './github';
18 +import { NPM_REGISTRY, REGISTRY_LANES, Language, type Language as Lang } from './lanes';
19 +import { discoverPythonPackages } from './python-metadata';
20 +import {
21 + appendFile,
22 + ensure,
23 + fail,
24 + parseJson,
25 + readJson,
26 + run,
27 + runMain,
28 + sha256Hex,
29 + withoutReleaseTokens,
30 +} from './runtime';
31 +import { BATCH_PATH, ReleaseBatch, batchLanguages } from './source';
32 +
33 +/** One reviewed distribution file: an npm tarball or a Python wheel/sdist. */
34 +export interface Distribution {
35 + readonly language: Lang;
36 + readonly name: string;
37 + readonly version: string;
38 + readonly file: string;
39 + readonly bytes: Uint8Array;
40 + readonly sha256: string;
41 +}
42 +
43 +export interface StableTag {
44 + readonly name: string;
45 + readonly version: string;
46 +}
47 +
48 +/** What the registries already serve, what they are missing, and which `latest` tags lag. */
49 +export interface RegistryState {
50 + readonly existing: ReadonlyArray<Distribution>;
51 + readonly missing: ReadonlyArray<Distribution>;
52 + readonly tagsToRepair: ReadonlyArray<StableTag>;
53 +}
54 +
55 +const REGISTRY_TIMEOUT = '30 seconds';
56 +
57 +const Url = Schema.String.check(
58 + Schema.makeFilter(value => (URL.canParse(value) ? undefined : 'Expected a URL'))
59 +);
60 +
61 +const isSuccess = (status: number) => status >= 200 && status < 300;
62 +
63 +const registryGet = (url: string) =>
64 + Effect.gen(function* () {
65 + const client = yield* HttpClient.HttpClient;
66 + return yield* client.get(url);
67 + }).pipe(Effect.provideService(FetchHttpClient.RequestInit, { redirect: 'error' }));
68 +
69 +/** A registry that does not yet serve something it should; propagation may still fix it. */
70 +class RegistryIncomplete extends Data.TaggedError('RegistryIncomplete')<{
71 + readonly message: string;
72 +}> {}
73 +const incomplete = (condition: boolean, message: string) =>
74 + condition ? Effect.void : Effect.fail(new RegistryIncomplete({ message }));
75 +
76 +/** Registry metadata at `url`, or `None` when the registry answers 404. */
77 +const registryJson = <S extends Schema.Top>(url: string, schema: S) =>
78 + Effect.gen(function* () {
79 + const response = yield* registryGet(url);
80 + if (response.status === 404) return Option.none<S['Type']>();
81 + yield* incomplete(
82 + isSuccess(response.status),
83 + `Registry lookup incomplete: ${url} (${response.status})`
84 + );
85 + return Option.some(yield* parseJson(schema, yield* response.text));
86 + }).pipe(Effect.timeout(REGISTRY_TIMEOUT));
87 +
88 +const registryBytes = (url: string, failure: (status: number) => string) =>
89 + Effect.gen(function* () {
90 + const response = yield* registryGet(url);
91 + yield* incomplete(isSuccess(response.status), failure(response.status));
92 + return new Uint8Array(yield* response.arrayBuffer);
93 + }).pipe(Effect.timeout(REGISTRY_TIMEOUT));
94 +
95 +const servedBy = (language: Lang, url: string) =>
96 + URL.canParse(url) &&
97 + new URL(url).protocol === 'https:' &&
98 + new URL(url).hostname === REGISTRY_LANES[language].downloadHost;
99 +
100 +/** A file the registry already serves must be byte for byte the reviewed candidate. */
101 +const verifyOriginalBytes = (url: string, file: Distribution) =>
102 + Effect.gen(function* () {
103 + yield* ensure(servedBy(file.language, url), `Unexpected registry download host: ${file.file}`);
104 + const bytes = yield* registryBytes(
105 + url,
106 + status => `Registry download incomplete: ${file.file} (${status})`
107 + );
108 + yield* ensure(
109 + bytes.length === file.bytes.length && (yield* sha256Hex(bytes)) === file.sha256,
110 + `Registry content conflict: ${file.name}@${file.version}/${file.file} differs from the approved candidate. Published files are never replaced; release a new version.`
111 + );
112 + });
113 +
114 +const PypiVersion = Schema.Struct({
115 + urls: Schema.Array(
116 + Schema.Struct({
117 + filename: Schema.String,
118 + url: Url,
119 + digests: Schema.Struct({ sha256: Schema.String }),
120 + })
121 + ),
122 +});
123 +const NpmVersion = Schema.Struct({
124 + name: Schema.String,
125 + version: Schema.String,
126 + dist: Schema.Struct({ tarball: Url }),
127 +});
128 +const NpmLatest = Schema.Struct({ version: Schema.String });
129 +const PackageIdentity = Schema.Struct({ name: Schema.String, version: Schema.String });
130 +
131 +const pythonDistributionNames = (name: string, version: string) => {
132 + const prefix = `${name.replaceAll('-', '_')}-${version}`;
133 + return [`${prefix}-py3-none-any.whl`, `${prefix}.tar.gz`];
134 +};
135 +
136 +interface LoadedCandidate {
137 + readonly cwd: string;
138 + readonly batch: ReleaseBatch;
139 + readonly directory: string;
140 + readonly manifest: CandidateManifest;
141 + readonly bytes: ReadonlyMap<string, Uint8Array>;
142 +}
143 +
144 +const distributionOf = (
145 + language: Lang,
146 + identity: { name: string; version: string },
147 + entry: CandidateFile,
148 + bytes: ReadonlyMap<string, Uint8Array>
149 +): Distribution => ({
150 + language,
151 + name: identity.name,
152 + version: identity.version,
153 + file: entry.name,
154 + bytes: bytes.get(entry.name) ?? new Uint8Array(),
155 + sha256: entry.sha256,
156 +});
157 +
158 +/** npm: one tarball per package version and a stable `latest` tag. */
159 +const npmAdapter = {
160 + reconcileGroup: (group: ReadonlyArray<Distribution>) =>
161 + Effect.gen(function* () {
162 + yield* ensure(group.length === 1, 'Expected exactly one npm tarball per package');
163 + const [file] = group;
164 + const metadata = yield* registryJson(
165 + REGISTRY_LANES.typescript.versionUrl(file.name, file.version),
166 + NpmVersion
167 + );
168 + if (Option.isSome(metadata)) {
169 + const { name, version, dist } = metadata.value;
170 + yield* ensure(
171 + name === file.name && version === file.version,
172 + 'Registry npm identity conflict'
173 + );
174 + yield* verifyOriginalBytes(dist.tarball, file);
175 + }
176 + const latest = yield* registryJson(
177 + `${NPM_REGISTRY}/${encodeURIComponent(file.name)}/latest`,
178 + NpmLatest
179 + );
180 + const tagged = Option.getOrUndefined(Option.map(latest, value => value.version));
181 + yield* ensure(
182 + tagged === undefined || semver.valid(tagged) !== null,
183 + `Invalid stable npm tag: ${file.name}`
184 + );
185 + return {
186 + existing: Option.isSome(metadata) ? [file] : [],
187 + missing: Option.isSome(metadata) ? [] : [file],
188 + tagsToRepair:
189 + tagged === undefined || semver.lt(tagged, file.version)
190 + ? [{ name: file.name, version: file.version }]
191 + : [],
192 + } satisfies RegistryState;
193 + }),
194 + candidateDistributions: (candidate: LoadedCandidate) =>
195 + Effect.gen(function* () {
196 + const path = yield* Path.Path;
197 + const releases = candidate.batch.plan.releases.filter(
198 + release => release.language === 'typescript'
199 + );
200 + const files: Distribution[] = [];
201 + for (const entry of candidate.manifest.files) {
202 + const manifest = yield* parseJson(
203 + PackageIdentity,
204 + yield* run(
205 + 'tar',
206 + ['-xOf', path.join(candidate.directory, entry.name), 'package/package.json'],
207 + { cwd: candidate.cwd, env: withoutReleaseTokens }
208 + )
209 + );
210 + yield* ensure(
211 + releases.some(
212 + release => release.name === manifest.name && release.newVersion === manifest.version
213 + ),
214 + 'Candidate contains an unplanned npm package'
215 + );
216 + files.push(distributionOf('typescript', manifest, entry, candidate.bytes));
217 + }
218 + yield* ensure(
219 + candidate.manifest.files.length === releases.length,
220 + 'Candidate npm package inventory differs'
221 + );
222 + return files;
223 + }),
224 +};
225 +
226 +/** PyPI: a wheel and sdist for every package of the synchronized Python train. */
227 +const pythonAdapter = {
228 + reconcileGroup: (group: ReadonlyArray<Distribution>) =>
229 + Effect.gen(function* () {
230 + const [first] = group;
231 + const metadata = yield* registryJson(
232 + REGISTRY_LANES.python.versionUrl(first.name, first.version),
233 + PypiVersion
234 + );
235 + const published = Option.match(metadata, { onNone: () => [], onSome: value => value.urls });
236 + yield* ensure(
237 + new Set(published.map(file => file.filename)).size === published.length &&
238 + published.every(file => group.some(candidate => candidate.file === file.filename)),
239 + `Unreviewed PyPI file for ${first.name}@${first.version}; release a new version`
240 + );
241 + const existing: Distribution[] = [];
242 + const missing: Distribution[] = [];
243 + for (const file of group) {
244 + const match = published.find(item => item.filename === file.file);
245 + if (!match) {
246 + missing.push(file);
247 + continue;
248 + }
249 + yield* ensure(
250 + match.digests.sha256 === file.sha256,
251 + `Registry content conflict: ${file.file}`
252 + );
253 + yield* verifyOriginalBytes(match.url, file);
254 + existing.push(file);
255 + }
256 + return { existing, missing, tagsToRepair: [] } satisfies RegistryState;
257 + }),
258 + candidateDistributions: (candidate: LoadedCandidate) =>
259 + Effect.gen(function* () {
260 + const release = candidate.batch.plan.releases.find(entry => entry.language === 'python');
261 + const version = release?.newVersion ?? '';
262 + const files: Distribution[] = [];
263 + for (const pkg of yield* discoverPythonPackages(candidate.cwd))
264 + for (const name of pythonDistributionNames(pkg.name, version)) {
265 + const entry = candidate.manifest.files.find(file => file.name === name);
266 + if (!entry) return yield* fail(`Original Python distribution missing: ${name}`);
267 + files.push(distributionOf('python', { name: pkg.name, version }, entry, candidate.bytes));
268 + }
269 + yield* ensure(
270 + candidate.manifest.files.length === files.length,
271 + 'Candidate Python file inventory differs'
272 + );
273 + return files;
274 + }),
275 +};
276 +
277 +const REGISTRY_ADAPTERS = { typescript: npmAdapter, python: pythonAdapter } as const;
278 +
279 +/** Splits reviewed files into those the registry already serves and those still missing. */
280 +export const reconcileRegistry = (files: ReadonlyArray<Distribution>) =>
281 + Effect.gen(function* () {
282 + const groups = Map.groupBy(files, file => `${file.language}:${file.name}@${file.version}`);
283 + const states: ReadonlyArray<RegistryState> = yield* Effect.forEach(groups.values(), group =>
284 + REGISTRY_ADAPTERS[group[0].language].reconcileGroup(group)
285 + );
286 + return {
287 + existing: states.flatMap(state => state.existing),
288 + missing: states.flatMap(state => state.missing),
289 + tagsToRepair: states.flatMap(state => state.tagsToRepair),
290 + } satisfies RegistryState;
291 + });
292 +
293 +const incompleteInventory = (state: RegistryState, stableTags: boolean) =>
294 + [
295 + ...state.missing.map(file => `${file.name}@${file.version}/${file.file}`),
296 + ...(stableTags ? state.tagsToRepair : []).map(tag => `${tag.name}@${tag.version} stable tag`),
297 + ].join(', ');
298 +
299 +/**
300 + * Polls the registries until every file is visible, and every stable tag too unless
301 + * `stableTags` is false (right after an upload, before `latest` moves).
302 + */
303 +export const waitForRegistry = (
304 + files: ReadonlyArray<Distribution>,
305 + options: {
306 + readonly attempts?: number;
307 + readonly spacing?: Duration.Input;
308 + readonly stableTags?: boolean;
309 + } = {}
310 +) => {
311 + const attempts = options.attempts ?? 6;
312 + return reconcileRegistry(files).pipe(
313 + Effect.flatMap(state => {
314 + const missing = incompleteInventory(state, options.stableTags ?? true);
315 + return missing
316 + ? Effect.fail(new RegistryIncomplete({ message: missing }))
317 + : Effect.succeed(state);
318 + }),
319 + Effect.retry({
320 + schedule: Schedule.spaced(options.spacing ?? '10 seconds'),
321 + times: attempts - 1,
322 + while: error => error instanceof RegistryIncomplete,
323 + }),
324 + Effect.catchTag('RegistryIncomplete', error =>
325 + Effect.fail(
326 + new RegistryIncomplete({
327 + message: `Registry verification incomplete after ${attempts} attempts: ${error.message}`,
328 + })
329 + )
330 + )
331 + );
332 +};
333 +
334 +/**
335 + * Downloads the approved release-PR artifact pinned for `language` and checks its files
336 + * against the manifest the release-PR build recorded.
337 + */
338 +export const loadOriginalDistributions = (
339 + cwd: string,
340 + batch: ReleaseBatch,
341 + sha: string,
342 + approved: ApprovedCandidates,
343 + language: Lang
344 +) =>
345 + Effect.gen(function* () {
346 + const fs = yield* FileSystem.FileSystem;
347 + const path = yield* Path.Path;
348 + const pinned = approved[language];
349 + if (pinned === undefined)
350 + return yield* fail(`No approved ${language} candidate was pinned for this run`);
351 + const listed = yield* request(
352 + 'GET',
353 + `${REPOSITORY_PATH}/actions/artifacts/${pinned.artifactId}`,
354 + ActionArtifact
355 + );
356 + const reused = yield* reuseCandidate(
357 + [listed],
358 + candidateArtifactName(language),
359 + pinned.run,
360 + sha
361 + );
362 + if (Option.isNone(reused) || reused.value.digest !== pinned.digest)
363 + return yield* fail(
364 + `The approved ${language} candidate artifact ${pinned.artifactId} changed; release a new version`
365 + );
366 + const directory = path.resolve(cwd, 'sdk-publication', language);
367 + yield* fs.remove(directory, { recursive: true, force: true });
368 + yield* downloadCandidate(reused.value, directory);
369 + const manifest = yield* readJson(CandidateManifest, path.join(directory, CANDIDATE_MANIFEST));
370 + yield* ensure(
371 + manifest.sourceSha === sha &&
372 + manifest.producerRun === pinned.run &&
373 + manifest.language === language,
374 + 'Original candidate producer/source conflict'
375 + );
376 + const bytes = yield* readCandidateFiles(directory);
377 + yield* verifyCandidateFiles(bytes, manifest.files);
378 + return yield* REGISTRY_ADAPTERS[language].candidateDistributions({
379 + cwd,
380 + batch,
381 + directory,
382 + manifest,
383 + bytes,
384 + });
385 + });
386 +
387 +/** Publishes under a release-only dist-tag, then moves `latest` once the upload is visible. */
388 +const publishNpm = (cwd: string, batch: ReleaseBatch, files: ReadonlyArray<Distribution>) =>
389 + Effect.gen(function* () {
390 + const path = yield* Path.Path;
391 + const npm = (args: ReadonlyArray<string>) =>
392 + run('npm', args, { cwd, env: withoutReleaseTokens });
393 + for (const file of (yield* reconcileRegistry(files)).missing)
394 + yield* npm([
395 + 'publish',
396 + path.resolve(cwd, 'sdk-publication/typescript', file.file),
397 + '--ignore-scripts',
398 + '--provenance',
399 + '--access=public',
400 + `--tag=sdk-release-${batch.id}`,
401 + ]);
402 + const visible = yield* waitForRegistry(files, { stableTags: false });
403 + for (const tag of visible.tagsToRepair)
404 + yield* npm(['dist-tag', 'add', `${tag.name}@${tag.version}`, 'latest']);
405 + yield* waitForRegistry(files);
406 + });
407 +
408 +/** Stages only the missing Python files for the trusted-publisher upload step. */
409 +const stageMissingPython = (cwd: string, files: ReadonlyArray<Distribution>, output: string) =>
410 + Effect.gen(function* () {
411 + const fs = yield* FileSystem.FileSystem;
412 + const path = yield* Path.Path;
413 + const directory = path.resolve(cwd, 'sdk-publication/missing-python');
414 + yield* fs.remove(directory, { recursive: true, force: true });
415 + yield* fs.makeDirectory(directory, { recursive: true });
416 + const { missing } = yield* reconcileRegistry(files);
417 + for (const file of missing) yield* fs.writeFile(path.join(directory, file.file), file.bytes);
418 + yield* appendFile(output, `missing=${missing.length > 0}\n`);
419 + });
420 +
421 +const Environment = Config.all({
422 + token: Config.schema(Schema.NonEmptyString, 'SDK_READ_TOKEN'),
423 + sha: Config.schema(Sha, 'GITHUB_SHA'),
424 + ref: Config.String('GITHUB_REF'),
425 + output: Config.String('GITHUB_OUTPUT'),
426 + approved: Config.schema(Schema.fromJsonString(ApprovedCandidates), 'APPROVED_CANDIDATES'),
427 +});
428 +
429 +const Arguments = Schema.Tuple([Schema.Literals(['publish', 'verify']), Language]);
430 +
431 +/**
432 + * `registry.ts publish typescript` uploads to npm and moves `latest`; `publish python` stages
433 + * the missing PyPI files for the upload action; `verify <language>` waits until every file is
434 + * served.
435 + */
436 +const main = Effect.gen(function* () {
437 + const [mode, language] = yield* Schema.decodeUnknownEffect(Arguments)(
438 + yield* (yield* Stdio.Stdio).args
439 + );
440 + const env = yield* Environment;
441 + yield* ensure(
442 + env.ref.startsWith('refs/tags/sdk-release/'),
443 + 'Registry jobs require a release tag'
444 + );
445 + const cwd = (yield* Path.Path).resolve('.');
446 + const batch = yield* readJson(ReleaseBatch, BATCH_PATH);
447 + yield* ensure(batchLanguages(batch).has(language), `This release plans no ${language} packages`);
448 + const files = yield* loadOriginalDistributions(cwd, batch, env.sha, env.approved, language).pipe(
449 + Effect.provide(GitHub.layer(env.token))
450 + );
451 + if (mode === 'verify') return yield* waitForRegistry(files);
452 + if (language === 'typescript') return yield* publishNpm(cwd, batch, files);
453 + yield* stageMissingPython(cwd, files, env.output);
454 +});
455 +
456 +if (import.meta.main) runMain(main);
scripts/sdk-release/report.ts new
+97
@@ -0,0 +1,97 @@
1 +import { Config, Console, Effect, Schema, Stdio } from 'effect';
2 +import semver from 'semver';
3 +import { GitHub, RunId, commentIssue, jobs, runUrl } from './github';
4 +import { packageLabel } from './release-pr';
5 +import { appendFile, readJson, runMain } from './runtime';
6 +import { BATCH_PATH, ReleaseBatch } from './source';
7 +
8 +/**
9 + * Posts the published versions on the request issue and returns the released npm versions,
10 + * which the docs job pins.
11 + */
12 +export const reportRelease = (batch: ReleaseBatch, runId: string) =>
13 + Effect.gen(function* () {
14 + yield* commentIssue(
15 + batch.issue,
16 + [
17 + `Published by ${runUrl(runId)}:`,
18 + '',
19 + ...batch.plan.releases.map(
20 + release => `- ${packageLabel(release.name)} ${release.newVersion}`
21 + ),
22 + ].join('\n')
23 + );
24 + return Object.fromEntries(
25 + batch.plan.releases
26 + .filter(release => release.language === 'typescript')
27 + .map(release => [release.name, release.newVersion])
28 + );
29 + });
30 +
31 +/** Posts the failed jobs of this run attempt and the recovery step on the request issue. */
32 +export const reportFailure = (batch: ReleaseBatch, runId: string) =>
33 + Effect.gen(function* () {
34 + const failed = (yield* jobs(runId, 'latest'))
35 + .filter(job => job.conclusion === 'failure')
36 + .map(job => job.name);
37 + yield* commentIssue(
38 + batch.issue,
39 + [
40 + `The release run failed in: ${failed.join(', ')}.`,
41 + `Open ${runUrl(runId)} and choose "Re-run failed jobs". A re-run uploads only the files the registries are missing.`,
42 + 'If the approved candidates expired or a registry already serves different bytes, add changesets for the affected packages and request a new release; it moves them to the next version.',
43 + ].join('\n\n')
44 + );
45 + });
46 +
47 +/**
48 + * The `name@version` specs that move the docs to the released npm versions. Packages the docs do
49 + * not use, and pins already at or past the release (an older release's retry), are left alone.
50 + */
51 +export const docsPinSpecs = (
52 + released: Readonly<Record<string, string>>,
53 + devDependencies: Readonly<Record<string, string>>
54 +) =>
55 + Object.entries(released)
56 + .filter(([name, version]) => {
57 + const pinned = semver.minVersion(devDependencies[name] ?? '');
58 + return name in devDependencies && (pinned === null || semver.gt(version, pinned));
59 + })
60 + .map(([name, version]) => `${name}@${version}`);
61 +
62 +const DocsManifest = Schema.Struct({
63 + devDependencies: Schema.Record(Schema.String, Schema.String),
64 +});
65 +
66 +/** `report.ts docs-pins <docs package.json>` prints the specs for `bun add`; reads `RELEASED`. */
67 +const docsPins = (manifestPath: string) =>
68 + Effect.gen(function* () {
69 + const released = yield* Config.schema(
70 + Schema.fromJsonString(Schema.Record(Schema.String, Schema.String)),
71 + 'RELEASED'
72 + );
73 + const { devDependencies } = yield* readJson(DocsManifest, manifestPath);
74 + yield* Console.log(docsPinSpecs(released, devDependencies).join(' '));
75 + });
76 +
77 +/** `report.ts released` or `report.ts failed`, run from the tag checkout. */
78 +const main = Effect.gen(function* () {
79 + const args = yield* (yield* Stdio.Stdio).args;
80 + if (args[0] === 'docs-pins') return yield* docsPins(args[1] ?? '');
81 + const [mode] = yield* Schema.decodeUnknownEffect(
82 + Schema.Tuple([Schema.Literals(['released', 'failed'])])
83 + )(args);
84 + const env = yield* Config.all({
85 + token: Config.schema(Schema.NonEmptyString, 'GITHUB_TOKEN'),
86 + runId: Config.schema(RunId, 'GITHUB_RUN_ID'),
87 + output: Config.String('GITHUB_OUTPUT'),
88 + });
89 + const batch = yield* readJson(ReleaseBatch, BATCH_PATH);
90 + yield* Effect.gen(function* () {
91 + if (mode === 'failed') return yield* reportFailure(batch, env.runId);
92 + const npm = yield* reportRelease(batch, env.runId);
93 + if (Object.keys(npm).length) yield* appendFile(env.output, `npm=${JSON.stringify(npm)}\n`);
94 + }).pipe(Effect.provide(GitHub.layer(env.token)));
95 +});
96 +
97 +if (import.meta.main) runMain(main);
test/sdk-release-approval.test.ts new
+137
@@ -0,0 +1,137 @@
1 +import { describe, expect, test } from 'bun:test';
2 +import { eligibleApproval } from '../scripts/sdk-release/approval';
3 +import { authorizeSourceTag } from '../scripts/sdk-release/batch';
4 +import {
5 + APP_SLUG,
6 + base,
7 + fakeController,
8 + head,
9 + merge,
10 + type ControllerOptions,
11 +} from './sdk-release-controller-fixture';
12 +import { runRelease } from './sdk-release-harness';
13 +
14 +const tag = (fetcher: ReturnType<typeof fakeController>['fetcher']) =>
15 + runRelease(authorizeSourceTag(9, APP_SLUG), fetcher);
16 +const tagWrite = {
17 + method: 'POST',
18 + path: '/git/refs',
19 + body: { ref: 'refs/tags/sdk-release/8', sha: head },
20 +};
21 +const gateComments = ['/issues/9/comments', '/issues/8/comments'];
22 +const commentBodies = (writes: ReadonlyArray<{ body: unknown }>) =>
23 + writes.map(write => (write.body as { body: string }).body);
24 +
25 +describe('release approval rule', () => {
26 + const excluded = ['author', 'requester'];
27 + const review = { user: 'reviewer', state: 'APPROVED', commitId: head, permission: 'write' };
28 +
29 + test.each([
30 + ['a writer who is neither author nor requester', [review], true],
31 + [
32 + 'a writer, despite a reader requesting changes',
33 + [review, { ...review, user: 'observer', state: 'CHANGES_REQUESTED', permission: 'read' }],
34 + true,
35 + ],
36 + ['the requester', [{ ...review, user: 'requester' }], false],
37 + ['the author', [{ ...review, user: 'author' }], false],
38 + ['an approval of an older commit', [{ ...review, commitId: base }], false],
39 + ['a reader', [{ ...review, permission: 'read' }], false],
40 + ['an approval later overridden', [review, { ...review, state: 'CHANGES_REQUESTED' }], false],
41 + [
42 + 'an approval while another writer requests changes',
43 + [review, { ...review, user: 'maintainer', state: 'CHANGES_REQUESTED' }],
44 + false,
45 + ],
46 + ])('%s → %p', (_name, reviews, expected) => {
47 + expect(eligibleApproval(reviews, excluded, head)).toBe(expected);
48 + });
49 +});
50 +
51 +describe('source tag', () => {
52 + test.each(['merge', 'squash'] as const)(
53 + 'a merged, approved release tags the reviewed head (%s merge)',
54 + async mergeKind => {
55 + const { fetcher, writes } = fakeController({ mergeKind });
56 + expect(await tag(fetcher)).toBe('sdk-release/8');
57 + expect(writes).toEqual([tagWrite]);
58 + }
59 + );
60 +
61 + test('required checks on a later page of more than 100 check runs still tag the head', async () => {
62 + const { fetcher, writes } = fakeController();
63 + const unrelated = Array.from({ length: 100 }, (_, index) => ({
64 + name: `matrix job ${index}`,
65 + head_sha: head,
66 + app: { slug: 'github-actions' },
67 + conclusion: 'success',
68 + }));
69 + const paged: typeof fetcher = (input, init) => {
70 + const url = new URL(String(input));
71 + if (!url.pathname.endsWith('/check-runs')) return fetcher(input, init);
72 + return (url.searchParams.get('page') ?? '1') === '1'
73 + ? Promise.resolve(Response.json({ total_count: 103, check_runs: unrelated }))
74 + : fetcher(input, init);
75 + };
76 + expect(await tag(paged)).toBe('sdk-release/8');
77 + expect(writes).toEqual([tagWrite]);
78 + });
79 +
80 + test('a release PR closed without merging creates no tag', async () => {
81 + const { fetcher, writes } = fakeController({ closedUnmerged: true });
82 + await expect(tag(fetcher)).rejects.toThrow('Release PR has not merged');
83 + expect(writes).toEqual([]);
84 + });
85 +
86 + test.each<[string, ControllerOptions, string]>([
87 + ['the requester approves', { reviewer: 'requester' }, 'lacks an approval'],
88 + ['the approval is for an older head', { oldReview: true }, 'lacks an approval'],
89 + ['a human pushed the head', { headAuthor: 'writer' }, 'release App commit'],
90 + [
91 + 'a human committed a head that names the App as author',
92 + { headCommitter: 'writer' },
93 + 'release App commit',
94 + ],
95 + ['the head commit is unverified', { unverifiedHead: true }, 'release App commit'],
96 + ['the head is not on the planned source', { headParent: merge }, 'release App commit'],
97 + ['the candidate build failed', { failedCandidate: true }, 'SDK candidate (typescript)'],
98 + ['the smoke test failed', { failedSmoke: true }, 'SDK candidate smoke (typescript)'],
99 + ['the docs check failed', { docs: 'failure' }, 'docs check'],
100 + ['the docs check is pending', { docs: 'pending' }, 'docs check'],
101 + ['the docs check is missing', { docs: 'missing' }, 'docs check'],
102 + ])(
103 + 'when %s, no tag is created and the PR and request are told to revert',
104 + async (_name, options, reason) => {
105 + const { fetcher, writes } = fakeController(options);
106 + await expect(tag(fetcher)).rejects.toThrow(reason);
107 + expect(writes.map(write => write.path)).toEqual(gateComments);
108 + for (const body of commentBodies(writes)) {
109 + expect(body).toContain(reason);
110 + expect(body).toContain('Revert this release PR on next');
111 + }
112 + }
113 + );
114 +
115 + test('a GitHub error during the gate asks for a re-run, never a revert', async () => {
116 + const { fetcher, writes } = fakeController();
117 + const unavailable: typeof fetcher = (input, init) =>
118 + new URL(String(input)).pathname.includes('/check-runs')
119 + ? Promise.resolve(new Response('Bad Gateway', { status: 502 }))
120 + : fetcher(input, init);
121 + await expect(tag(unavailable)).rejects.toThrow('(502)');
122 + expect(writes.map(write => write.path)).toEqual(gateComments);
123 + for (const body of commentBodies(writes)) {
124 + expect(body).not.toContain('Revert');
125 + expect(body).toContain('Re-run failed jobs');
126 + }
127 + });
128 +
129 + test('an existing matching tag is reused and a tag on another commit is refused', async () => {
130 + const matching = fakeController({ existingTag: head });
131 + expect(await tag(matching.fetcher)).toBe('sdk-release/8');
132 + expect(matching.writes).toEqual([]);
133 + const moved = fakeController({ existingTag: merge });
134 + await expect(tag(moved.fetcher)).rejects.toThrow('another source');
135 + expect(moved.writes).toEqual([]);
136 + });
137 +});
test/sdk-release-approved-candidate.test.ts new
+210
@@ -0,0 +1,210 @@
1 +import { afterAll, expect, test } from 'bun:test';
2 +import { adoptApprovedCandidate } from '../scripts/sdk-release/candidates';
3 +import { formatJson, run, sha256Hex } from '../scripts/sdk-release/runtime';
4 +import {
5 + json,
6 + readText,
7 + runRelease,
8 + temporaryDirectories,
9 + writeText,
10 + type Fetch,
11 +} from './sdk-release-harness';
12 +
13 +const directories = temporaryDirectories('sdk-approved-candidate-');
14 +afterAll(() => directories.cleanup());
15 +
16 +const head = 'a'.repeat(40);
17 +const wheel = 'composio-1.0.1-py3-none-any.whl';
18 +const wheelText = 'approved wheel bytes';
19 +
20 +/** A real candidate archive as the release-PR run uploaded it; `listed` is the manifest's text. */
21 +const approvedArchive = async (
22 + listed = wheelText,
23 + identity: Record<string, unknown> = { producerRun: 41 }
24 +) => {
25 + const directory = await directories.create();
26 + const listedBytes = new TextEncoder().encode(listed);
27 + await writeText(`${directory}/${wheel}`, wheelText);
28 + await writeText(
29 + `${directory}/candidate.json`,
30 + formatJson({
31 + sourceSha: head,
32 + language: 'python',
33 + ...identity,
34 + files: [
35 + { name: wheel, sha256: await runRelease(sha256Hex(listedBytes)), size: listedBytes.length },
36 + ],
37 + })
38 + );
39 + const zip = `${directory}/candidate.zip`;
40 + await runRelease(
41 + run('zip', ['-j', '-q', zip, `${directory}/${wheel}`, `${directory}/candidate.json`])
42 + );
43 + const bytes = new Uint8Array(await Bun.file(zip).arrayBuffer());
44 + return { bytes, digest: `sha256:${await runRelease(sha256Hex(bytes))}` };
45 +};
46 +
47 +const releaseRun = (id: number, conclusion = 'success') => ({
48 + id,
49 + path: '.github/workflows/sdk.release.candidates.yml',
50 + head_branch: 'sdk-release/batch-8',
51 + head_sha: head,
52 + event: 'pull_request',
53 + repository: { full_name: 'ComposioHQ/composio' },
54 + status: 'completed',
55 + conclusion,
56 + created_at: '2026-11-03T00:00:00Z',
57 + run_attempt: 1,
58 + html_url: `https://github.com/ComposioHQ/composio/actions/runs/${id}`,
59 +});
60 +
61 +const fakeGitHub =
62 + (
63 + archive: { bytes: Uint8Array<ArrayBuffer>; digest: string },
64 + runs: unknown[],
65 + expired = false,
66 + build: {
67 + createdAt?: string;
68 + conclusion?: string;
69 + completedAt?: string;
70 + smokeOnlyRerun?: boolean;
71 + } = {}
72 + ): Fetch =>
73 + async input => {
74 + const url = new URL(String(input));
75 + if (url.hostname === 'artifacts.test') return new Response(archive.bytes);
76 + const path = url.pathname.replace('/repos/ComposioHQ/composio', '');
77 + if (path === '/actions/workflows/sdk.release.candidates.yml/runs')
78 + return json({ workflow_runs: runs });
79 + if (path === '/actions/runs/41/jobs') {
80 + const buildJob = {
81 + name: 'SDK candidate (python)',
82 + status: 'completed',
83 + conclusion: build.conclusion ?? 'success',
84 + started_at: '2026-11-03T00:01:00Z',
85 + completed_at: build.completedAt ?? '2026-11-03T00:10:00Z',
86 + steps: [
87 + {
88 + name: 'Retain candidate bytes for publication',
89 + conclusion: 'success',
90 + started_at: '2026-11-03T00:04:00Z',
91 + completed_at: '2026-11-03T00:06:00Z',
92 + },
93 + ],
94 + };
95 + const smoke = (conclusion: string, startedAt: string, completedAt: string) => ({
96 + name: 'SDK candidate smoke (python)',
97 + status: 'completed',
98 + conclusion,
99 + started_at: startedAt,
100 + completed_at: completedAt,
101 + });
102 + if (!build.smokeOnlyRerun)
103 + return json({
104 + jobs: [buildJob, smoke('success', '2026-11-03T00:11:00Z', '2026-11-03T00:20:00Z')],
105 + });
106 + // "Re-run failed jobs" on the smoke job: the latest attempt holds only the smoke job.
107 + const latest = [smoke('success', '2026-11-03T01:00:00Z', '2026-11-03T01:10:00Z')];
108 + return json({
109 + jobs:
110 + url.searchParams.get('filter') === 'latest'
111 + ? latest
112 + : [
113 + buildJob,
114 + smoke('failure', '2026-11-03T00:11:00Z', '2026-11-03T00:20:00Z'),
115 + ...latest,
116 + ],
117 + });
118 + }
119 + if (path === '/actions/runs/41/artifacts')
120 + return json({
121 + artifacts: [
122 + {
123 + id: 77,
124 + name: 'sdk-candidate-python',
125 + digest: archive.digest,
126 + expired,
127 + created_at: build.createdAt ?? '2026-11-03T00:05:00Z',
128 + expires_at: '2099-12-01T00:00:00Z',
129 + workflow_run: { id: 41, head_sha: head },
130 + },
131 + ],
132 + });
133 + if (path === '/actions/artifacts/77/zip')
134 + return new Response(null, {
135 + status: 302,
136 + headers: { location: 'https://artifacts.test/77.zip' },
137 + });
138 + throw new Error(`Unexpected read ${path}`);
139 + };
140 +
141 +test('a source-tag run copies the newest approved release-PR candidate byte for byte', async () => {
142 + const archive = await approvedArchive();
143 + const target = `${await directories.create()}/python`;
144 + const approved = await runRelease(
145 + adoptApprovedCandidate(head, 'python', target),
146 + fakeGitHub(archive, [releaseRun(40), releaseRun(41), releaseRun(42, 'failure')])
147 + );
148 + expect(approved).toEqual({ run: 41, artifactId: 77, digest: archive.digest });
149 + expect(await readText(`${target}/${wheel}`)).toBe(wheelText);
150 + expect(await Bun.file(`${target}/candidate.json`).exists()).toBe(false);
151 +});
152 +
153 +test('approved files that differ from their recorded checksums are not adopted', async () => {
154 + const archive = await approvedArchive('different wheel bytes');
155 + const target = `${await directories.create()}/python`;
156 + await expect(
157 + runRelease(
158 + adoptApprovedCandidate(head, 'python', target),
159 + fakeGitHub(archive, [releaseRun(41)])
160 + )
161 + ).rejects.toThrow(`Candidate checksum mismatch: ${wheel}`);
162 +});
163 +
164 +test.each<
165 + [string, Parameters<typeof fakeGitHub>[1], boolean, Parameters<typeof fakeGitHub>[3], string]
166 +>([
167 + [
168 + 'no successful release-PR run',
169 + [releaseRun(41, 'failure')],
170 + false,
171 + {},
172 + 'No successful release-PR run built the approved python candidate',
173 + ],
174 + ['an expired artifact', [releaseRun(41)], true, {}, 'Original candidate unavailable'],
175 + [
176 + 'an artifact created after the upload step',
177 + [releaseRun(41)],
178 + false,
179 + { createdAt: '2026-11-03T00:08:00Z' },
180 + 'sdk-candidate-python was created at 2026-11-03T00:08:00Z, outside its upload step',
181 + ],
182 + [
183 + 'a build job that did not succeed',
184 + [releaseRun(41)],
185 + false,
186 + { conclusion: 'skipped' },
187 + 'The SDK candidate (python) job did not upload a candidate',
188 + ],
189 +])('refuses to adopt the candidate of %s', async (_name, runs, expired, build, message) => {
190 + const archive = await approvedArchive();
191 + const target = `${await directories.create()}/python`;
192 + await expect(
193 + runRelease(
194 + adoptApprovedCandidate(head, 'python', target),
195 + fakeGitHub(archive, runs, expired, build)
196 + )
197 + ).rejects.toThrow(message);
198 + expect(await Bun.file(`${target}/${wheel}`).exists()).toBe(false);
199 +});
200 +
201 +test('a smoke-only re-run attempt still adopts the candidate its first attempt built', async () => {
202 + const archive = await approvedArchive();
203 + const target = `${await directories.create()}/python`;
204 + const approved = await runRelease(
205 + adoptApprovedCandidate(head, 'python', target),
206 + fakeGitHub(archive, [{ ...releaseRun(41), run_attempt: 2 }], false, { smokeOnlyRerun: true })
207 + );
208 + expect(approved).toEqual({ run: 41, artifactId: 77, digest: archive.digest });
209 + expect(await readText(`${target}/${wheel}`)).toBe(wheelText);
210 +});
test/sdk-release-candidate-pin.test.ts new
+156
@@ -0,0 +1,156 @@
1 +import { afterEach, expect, test } from 'bun:test';
2 +import { formatJson, run, sha256Hex, spawn } from '../scripts/sdk-release/runtime';
3 +import {
4 + readText,
5 + resolvePath,
6 + runRelease,
7 + temporaryDirectories,
8 + writeText,
9 +} from './sdk-release-harness';
10 +import { batch } from './sdk-release-controller-fixture';
11 +
12 +const directories = temporaryDirectories('sdk-candidate-pin-');
13 +afterEach(() => directories.cleanup());
14 +
15 +const head = 'b'.repeat(40);
16 +const tarball = 'composio-core-1.0.1.tgz';
17 +
18 +/** The archive release-PR run 41 uploaded for `@composio/core@1.0.1`. */
19 +const approvedArchive = async () => {
20 + const directory = await directories.create();
21 + const bytes = new TextEncoder().encode('approved tarball bytes');
22 + await writeText(`${directory}/${tarball}`, 'approved tarball bytes');
23 + await writeText(
24 + `${directory}/candidate.json`,
25 + formatJson({
26 + sourceSha: head,
27 + producerRun: 41,
28 + language: 'typescript',
29 + files: [{ name: tarball, sha256: await runRelease(sha256Hex(bytes)), size: bytes.length }],
30 + })
31 + );
32 + const zip = `${directory}/candidate.zip`;
33 + await runRelease(
34 + run('zip', ['-j', '-q', zip, `${directory}/${tarball}`, `${directory}/candidate.json`])
35 + );
36 + const archive = new Uint8Array(await Bun.file(zip).arrayBuffer());
37 + return { zip, digest: `sha256:${await runRelease(sha256Hex(archive))}` };
38 +};
39 +
40 +/** The tag run's pinning job for tag run 9, run as the CLI on a fresh checkout. */
41 +const pinAttempt = async (archive: { zip: string; digest: string }) => {
42 + const directory = await directories.create();
43 + await writeText(`${directory}/package.json`, JSON.stringify({ name: 'root', private: true }));
44 + await writeText(`${directory}/pnpm-workspace.yaml`, "packages: ['core']\n");
45 + await writeText(
46 + `${directory}/core/package.json`,
47 + JSON.stringify({ name: '@composio/core', version: '1.0.1' })
48 + );
49 + await writeText(`${directory}/.github/sdk-release-plan.json`, JSON.stringify(batch));
50 + const routes = {
51 + '/actions/workflows/sdk.release.candidates.yml/runs': {
52 + workflow_runs: [
53 + {
54 + id: 41,
55 + path: '.github/workflows/sdk.release.candidates.yml',
56 + head_branch: 'sdk-release/batch-8',
57 + head_sha: head,
58 + event: 'pull_request',
59 + repository: { full_name: 'ComposioHQ/composio' },
60 + status: 'completed',
61 + conclusion: 'success',
62 + created_at: '2026-11-03T00:00:00Z',
63 + run_attempt: 1,
64 + html_url: 'https://github.com/ComposioHQ/composio/actions/runs/41',
65 + },
66 + ],
67 + },
68 + '/actions/runs/41/jobs': {
69 + jobs: [
70 + {
71 + name: 'SDK candidate (typescript)',
72 + status: 'completed',
73 + conclusion: 'success',
74 + started_at: '2026-11-03T00:01:00Z',
75 + completed_at: '2026-11-03T00:10:00Z',
76 + steps: [
77 + {
78 + name: 'Retain candidate bytes for publication',
79 + conclusion: 'success',
80 + started_at: '2026-11-03T00:04:00Z',
81 + completed_at: '2026-11-03T00:06:00Z',
82 + },
83 + ],
84 + },
85 + ],
86 + },
87 + '/actions/runs/41/artifacts': {
88 + artifacts: [
89 + {
90 + id: 77,
91 + name: 'sdk-candidate-typescript',
92 + digest: archive.digest,
93 + expired: false,
94 + created_at: '2026-11-03T00:05:00Z',
95 + expires_at: '2099-12-01T00:00:00Z',
96 + workflow_run: { id: 41, head_sha: head },
97 + },
98 + ],
99 + },
100 + };
101 + const preload = `${directory}/fetch.ts`;
102 + await writeText(
103 + preload,
104 + `const routes = ${JSON.stringify(routes)};
105 +globalThis.fetch = async input => {
106 + const url = new URL(String(input));
107 + if (url.hostname === 'artifacts.test') return new Response(Bun.file(${JSON.stringify(archive.zip)}));
108 + if (url.pathname.endsWith('/actions/artifacts/77/zip'))
109 + return new Response(null, { status: 302, headers: { location: 'https://artifacts.test/77.zip' } });
110 + const route = Object.keys(routes).find(key => url.pathname.endsWith(key));
111 + return route ? Response.json(routes[route]) : new Response('{}', { status: 502 });
112 +};
113 +`
114 + );
115 + const output = `${directory}/output`;
116 + await writeText(output, '');
117 + const summary = `${directory}/summary`;
118 + await writeText(summary, '');
119 + const result = await runRelease(
120 + spawn(
121 + 'bun',
122 + ['--preload', preload, await resolvePath('scripts/sdk-release/candidates.ts'), 'approve'],
123 + {
124 + cwd: directory,
125 + env: {
126 + GITHUB_RUN_ID: '9',
127 + GITHUB_SHA: head,
128 + GITHUB_REF: 'refs/tags/sdk-release/8',
129 + GITHUB_OUTPUT: output,
130 + GITHUB_STEP_SUMMARY: summary,
131 + SDK_READ_TOKEN: 'fixture',
132 + SDK_RELEASE_TOKEN: '',
133 + },
134 + }
135 + )
136 + );
137 + expect({ exitCode: result.exitCode, stderr: result.stderr }).toMatchObject({ exitCode: 0 });
138 + return {
139 + output: await readText(output),
140 + tarball: await readText(`${directory}/sdk-candidate/typescript/${tarball}`),
141 + };
142 +};
143 +
144 +test('the tag run pins the approved release-PR artifact and its languages as job outputs', async () => {
145 + const archive = await approvedArchive();
146 + const pinned = await pinAttempt(archive);
147 + expect(pinned.tarball).toBe('approved tarball bytes');
148 + expect(pinned.output).toBe(
149 + [
150 + `approved=${JSON.stringify({ typescript: { run: 41, artifactId: 77, digest: archive.digest } })}`,
151 + 'typescript=true',
152 + 'python=false',
153 + '',
154 + ].join('\n')
155 + );
156 +});
test/sdk-release-publication.test.ts new
+287
@@ -0,0 +1,287 @@
1 +import { describe, expect, test } from 'bun:test';
2 +import type { Language } from '../scripts/sdk-release/lanes';
3 +import {
4 + loadOriginalDistributions,
5 + reconcileRegistry as reconcileRegistryEffect,
6 + waitForRegistry,
7 + type Distribution,
8 +} from '../scripts/sdk-release/registry';
9 +import { docsPinSpecs, reportFailure, reportRelease } from '../scripts/sdk-release/report';
10 +import { sha256Hex } from '../scripts/sdk-release/runtime';
11 +import type { ReleaseBatch } from '../scripts/sdk-release/source';
12 +import { batch, head } from './sdk-release-controller-fixture';
13 +import { runRelease, type Fetch } from './sdk-release-harness';
14 +
15 +const distribution = async (file: string, language: Language = 'python'): Promise<Distribution> => {
16 + const bytes = new TextEncoder().encode(`original ${file}`);
17 + return {
18 + language,
19 + name: language === 'python' ? 'composio' : '@composio/core',
20 + version: '1.0.1',
21 + file,
22 + bytes,
23 + sha256: await runRelease(sha256Hex(bytes)),
24 + };
25 +};
26 +const wheel = await distribution('composio-1.0.1-py3-none-any.whl');
27 +const sdist = await distribution('composio-1.0.1.tar.gz');
28 +const tarball = await distribution('composio-core-1.0.1.tgz', 'typescript');
29 +const reconcileRegistry = (files: Distribution[], fetch: Fetch) =>
30 + runRelease(reconcileRegistryEffect(files), fetch);
31 +
32 +function registry(
33 + files: Distribution[],
34 + options: { conflict?: boolean; unknownPython?: boolean; newer?: boolean } = {}
35 +) {
36 + const reads: string[] = [];
37 + return {
38 + reads,
39 + fetcher: (async (input: string | URL | Request) => {
40 + const url = String(input);
41 + reads.push(url);
42 + const json = (value: unknown, status = 200) =>
43 + new Response(JSON.stringify(value), { status });
44 + if (
45 + url.startsWith('https://files.pythonhosted.org/') ||
46 + url.startsWith('https://registry.npmjs.org/files/')
47 + ) {
48 + const file = files.find(file => url.endsWith(file.file));
49 + return new Response(
50 + options.conflict ? 'wrong bytes' : file ? new Uint8Array(file.bytes).buffer : undefined,
51 + { status: file ? 200 : 404 }
52 + );
53 + }
54 + if (url.startsWith('https://pypi.org/'))
55 + return files.some(file => file.language === 'python')
56 + ? json({
57 + urls: [
58 + ...files
59 + .filter(file => file.language === 'python')
60 + .map(file => ({
61 + filename: file.file,
62 + url: `https://files.pythonhosted.org/${file.file}`,
63 + digests: { sha256: file.sha256 },
64 + })),
65 + ...(options.unknownPython
66 + ? [
67 + {
68 + filename: 'unreviewed.whl',
69 + url: 'https://files.pythonhosted.org/unreviewed.whl',
70 + digests: { sha256: 'f'.repeat(64) },
71 + },
72 + ]
73 + : []),
74 + ],
75 + })
76 + : json({}, 404);
77 + if (url.endsWith('/latest')) return json({ version: options.newer ? '1.0.2' : '1.0.1' });
78 + if (url.startsWith('https://registry.npmjs.org/'))
79 + return files.some(file => file.language === 'typescript')
80 + ? json({
81 + name: tarball.name,
82 + version: tarball.version,
83 + dist: { tarball: `https://registry.npmjs.org/files/${tarball.file}` },
84 + })
85 + : json({}, 404);
86 + throw new Error(`Unexpected registry fetch ${url}`);
87 + }) satisfies Fetch,
88 + };
89 +}
90 +
91 +describe('registry reconciliation', () => {
92 + test('a first run finds every candidate file missing', async () => {
93 + const result = await reconcileRegistry([tarball, wheel, sdist], registry([]).fetcher);
94 + expect(result.existing).toEqual([]);
95 + expect(result.missing.map(file => file.file)).toEqual([tarball.file, wheel.file, sdist.file]);
96 + });
97 + test('identical published files are accepted and only the missing sdist is left to upload', async () => {
98 + const fake = registry([wheel]);
99 + const result = await reconcileRegistry([wheel, sdist], fake.fetcher);
100 + expect(result.existing.map(file => file.file)).toEqual([wheel.file]);
101 + expect(result.missing.map(file => file.file)).toEqual([sdist.file]);
102 + expect(result.missing[0].bytes).toEqual(sdist.bytes);
103 + expect(fake.reads).toContain(`https://files.pythonhosted.org/${wheel.file}`);
104 + });
105 + test('a re-run after npm succeeded and PyPI failed leaves only the PyPI files to upload', async () => {
106 + const result = await reconcileRegistry([tarball, wheel, sdist], registry([tarball]).fetcher);
107 + expect(result.existing.map(file => file.file)).toEqual([tarball.file]);
108 + expect(result.missing.map(file => file.file)).toEqual([wheel.file, sdist.file]);
109 + expect(result.tagsToRepair).toEqual([]);
110 + });
111 + test('published bytes that differ from the candidate stop with a conflict naming the file', async () => {
112 + await expect(
113 + reconcileRegistry([wheel, sdist], registry([wheel], { conflict: true }).fetcher)
114 + ).rejects.toThrow(`Registry content conflict: composio@1.0.1/${wheel.file}`);
115 + await expect(
116 + reconcileRegistry([wheel, sdist], registry([wheel], { unknownPython: true }).fetcher)
117 + ).rejects.toThrow('Unreviewed');
118 + });
119 + test('Python-only operation does not require npm outputs or requests', async () => {
120 + const fake = registry([wheel, sdist]);
121 + expect((await reconcileRegistry([wheel, sdist], fake.fetcher)).missing).toHaveLength(0);
122 + expect(fake.reads.some(url => url.includes('npmjs'))).toBe(false);
123 + });
124 + test('older retries accept a newer stable tag without requesting a downgrade', async () => {
125 + const result = await reconcileRegistry([tarball], registry([tarball], { newer: true }).fetcher);
126 + expect(result.tagsToRepair).toEqual([]);
127 + });
128 + test('bounded propagation ends with the exact incomplete file inventory', async () => {
129 + const fake = registry([wheel]);
130 + await expect(
131 + runRelease(
132 + waitForRegistry([wheel, sdist], { attempts: 3, spacing: '0 millis' }),
133 + fake.fetcher
134 + )
135 + ).rejects.toThrow(`after 3 attempts: composio@1.0.1/${sdist.file}`);
136 + expect(fake.reads.filter(url => url.startsWith('https://pypi.org/'))).toHaveLength(3);
137 + });
138 + test('a fresh npm upload is polled until visible, then its stable tag is left to move', async () => {
139 + const fake = registry([tarball]);
140 + let lookups = 0;
141 + const lagging: Fetch = async input => {
142 + const url = String(input);
143 + if (url.endsWith('/latest')) return new Response(JSON.stringify({ version: '1.0.0' }));
144 + if (url.startsWith('https://registry.npmjs.org/') && lookups++ === 0)
145 + return new Response('{}', { status: 404 });
146 + return fake.fetcher(input);
147 + };
148 + const state = await runRelease(
149 + waitForRegistry([tarball], { attempts: 3, spacing: '0 millis', stableTags: false }),
150 + lagging
151 + );
152 + expect(state.missing).toEqual([]);
153 + expect(state.tagsToRepair).toEqual([{ name: tarball.name, version: tarball.version }]);
154 + });
155 + test('a content conflict fails at once instead of waiting for propagation', async () => {
156 + const fake = registry([wheel], { conflict: true });
157 + await expect(
158 + runRelease(
159 + waitForRegistry([wheel, sdist], { attempts: 3, spacing: '0 millis' }),
160 + fake.fetcher
161 + )
162 + ).rejects.toThrow(`Registry content conflict: composio@1.0.1/${wheel.file}`);
163 + expect(fake.reads.filter(url => url.startsWith('https://pypi.org/'))).toHaveLength(1);
164 + });
165 + test('published bytes of the same length but different content are a conflict', async () => {
166 + const fake = registry([wheel, sdist]);
167 + const sameLength: Fetch = async input => {
168 + const response = await fake.fetcher(input);
169 + if (!String(input).endsWith(wheel.file)) return response;
170 + const bytes = new Uint8Array(await response.arrayBuffer());
171 + bytes[0] ^= 0xff;
172 + return new Response(bytes);
173 + };
174 + await expect(reconcileRegistry([wheel, sdist], sameLength)).rejects.toThrow(
175 + `Registry content conflict: composio@1.0.1/${wheel.file}`
176 + );
177 + });
178 +});
179 +
180 +describe('pinned candidate download', () => {
181 + const pinned = { run: 41, artifactId: 77, digest: `sha256:${'c'.repeat(64)}` };
182 + const listed = (overrides: { digest?: string; run?: number }) => ({
183 + id: 77,
184 + name: 'sdk-candidate-typescript',
185 + digest: overrides.digest ?? pinned.digest,
186 + expired: false,
187 + created_at: '2026-11-03T00:05:00Z',
188 + expires_at: '2099-12-01T00:00:00Z',
189 + workflow_run: { id: overrides.run ?? 41, head_sha: head },
190 + });
191 + test.each([
192 + ['a different digest', { digest: `sha256:${'d'.repeat(64)}` }, 'artifact 77 changed'],
193 + ['another producer run', { run: 42 }, 'Candidate source or producer does not match'],
194 + ])('a pinned artifact with %s is refused before download', async (_name, overrides, message) => {
195 + const reads: string[] = [];
196 + const fetcher: Fetch = async input => {
197 + const path = new URL(String(input)).pathname;
198 + reads.push(path);
199 + if (path === '/repos/ComposioHQ/composio/actions/artifacts/77')
200 + return new Response(JSON.stringify(listed(overrides)));
201 + throw new Error(`Unexpected read ${path}`);
202 + };
203 + await expect(
204 + runRelease(
205 + loadOriginalDistributions('.', batch, head, { typescript: pinned }, 'typescript'),
206 + fetcher
207 + )
208 + ).rejects.toThrow(message);
209 + expect(reads).toEqual(['/repos/ComposioHQ/composio/actions/artifacts/77']);
210 + });
211 + test('a language without a pinned candidate is refused without any request', async () => {
212 + await expect(
213 + runRelease(loadOriginalDistributions('.', batch, head, {}, 'typescript'), async () => {
214 + throw new Error('no request expected');
215 + })
216 + ).rejects.toThrow('No approved typescript candidate was pinned');
217 + });
218 +});
219 +
220 +/** Records the comments a tag run posts on its request issue. */
221 +function reportingRun(jobs: ReadonlyArray<{ name: string; conclusion: string | null }> = []) {
222 + const comments: { issue: number; body: string }[] = [];
223 + const fetch: Fetch = async (input, init) => {
224 + const url = new URL(String(input));
225 + const path = url.pathname.replace('/repos/ComposioHQ/composio', '') + url.search;
226 + const respond = (data: unknown) => new Response(JSON.stringify(data));
227 + if (path === '/actions/runs/77/jobs?filter=latest&per_page=100&page=1')
228 + return respond({ jobs: jobs.map(job => ({ ...job, steps: [] })) });
229 + const comment = /^\/issues\/(\d+)\/comments$/.exec(path);
230 + if (comment && init?.method === 'POST') {
231 + comments.push({ issue: Number(comment[1]), body: JSON.parse(String(init.body)).body });
232 + return respond({ id: comments.length });
233 + }
234 + throw new Error(`Unexpected request: ${init?.method ?? 'GET'} ${path}`);
235 + };
236 + return { fetch, comments };
237 +}
238 +
239 +describe('release run reporting', () => {
240 + test('a published release lists every version and returns only the npm versions to pin', async () => {
241 + const mixed: ReleaseBatch = {
242 + ...batch,
243 + plan: {
244 + ...batch.plan,
245 + releases: [
246 + ...batch.plan.releases,
247 + {
248 + name: '@composio/python-sdk',
249 + type: 'patch',
250 + oldVersion: '0.31.0',
251 + newVersion: '0.31.1',
252 + changesets: [],
253 + language: 'python',
254 + },
255 + ],
256 + },
257 + };
258 + const run = reportingRun();
259 + expect(await runRelease(reportRelease(mixed, '77'), run.fetch)).toEqual({
260 + '@composio/core': '1.0.1',
261 + });
262 + expect(run.comments).toHaveLength(1);
263 + expect(run.comments[0].issue).toBe(8);
264 + expect(run.comments[0].body).toContain('`@composio/core` 1.0.1');
265 + expect(run.comments[0].body).toContain('Python SDK (`composio` and every provider) 0.31.1');
266 + });
267 + test('a failed run names only the failed jobs on the request issue', async () => {
268 + const run = reportingRun([
269 + { name: 'publish-npm', conclusion: 'success' },
270 + { name: 'publish-pypi', conclusion: 'failure' },
271 + { name: 'docs', conclusion: 'skipped' },
272 + ]);
273 + await runRelease(reportFailure(batch, '77'), run.fetch);
274 + expect(run.comments).toHaveLength(1);
275 + expect(run.comments[0].issue).toBe(8);
276 + expect(run.comments[0].body).toContain('failed in: publish-pypi.');
277 + });
278 +});
279 +
280 +test('docs pins move only used packages forward, so an older retry never downgrades them', () => {
281 + expect(
282 + docsPinSpecs(
283 + { '@composio/core': '0.29.0', '@composio/openai': '0.13.2', '@composio/unused': '1.0.0' },
284 + { '@composio/core': '^0.28.0', '@composio/openai': '0.13.3', react: '^19' }
285 + )
286 + ).toEqual(['@composio/core@0.29.0']);
287 +});