@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 446 lines 16.8 KB
Raw
1 import { Config, Effect, FileSystem, Option, Path, Schema, Stdio } from 'effect';
2 import { getPackages } from '@manypkg/get-packages';
3 import { isReleaseAppCommit } from './approval';
4 import {
5 ActionArtifact,
6 ApprovedCandidates,
7 CANDIDATE_MANIFEST,
8 type ApprovedCandidate,
9 CandidateManifest,
10 candidateArtifactName,
11 downloadCandidate,
12 normalizeArtifactDigest,
13 originalCandidates,
14 readCandidateFiles,
15 reuseCandidate,
16 verifyCandidateFiles,
17 type CandidateArtifact,
18 } from './artifacts';
19 import {
20 GitHub,
21 PositiveIntFromString,
22 REPOSITORY_PATH,
23 ReleaseRun,
24 SDK_RELEASE_REPOSITORY,
25 Sha,
26 jobs,
27 request,
28 } from './github';
29 import { Language, REGISTRY_VERSION } from './lanes';
30 import { applySdkReleasePlan, type SdkRelease } from './plan';
31 import { discoverPythonPackages } from './python-metadata';
32 import {
33 appendFile,
34 ensure,
35 fail,
36 formatJson,
37 fromPromise,
38 inDirectory,
39 readJson,
40 runMain,
41 sha256Hex,
42 type ReleaseServices,
43 } from './runtime';
44 import { BATCH_PATH, ReleaseBatch, batchLanguages, channelOf, releaseTag } from './source';
45
46 export const PackedManifest = Schema.StructWithRest(
47 Schema.Struct({ name: Schema.String, version: Schema.String }),
48 [Schema.Record(Schema.String, Schema.Unknown)]
49 );
50
51 export const tarballName = (name: string, version: string) =>
52 `${name.replace(/^@/, '').replaceAll('/', '-')}-${version}.tgz`;
53
54 /**
55 * Builds the SDK packages and packs the selected ones into `directory`. Provider compatibility is
56 * checked by ts.test.yml; this job runs no registry installs before the files are hashed.
57 */
58 export const buildTypescript = (cwd: string, directory: string, names: ReadonlySet<string>) =>
59 Effect.gen(function* () {
60 const path = yield* Path.Path;
61 const exec = inDirectory(cwd);
62 const { packages } = yield* fromPromise(() => getPackages(cwd));
63 const selected = packages.filter(pkg => names.has(pkg.packageJson.name));
64 yield* exec('pnpm', [
65 'exec',
66 'turbo',
67 'build',
68 ...selected.map(pkg => `--filter=${pkg.packageJson.name}`),
69 ]);
70 for (const pkg of selected) {
71 yield* exec('pnpm', [
72 '--dir',
73 pkg.dir,
74 'pack',
75 '--pack-destination',
76 directory,
77 '--config.ignore-scripts=true',
78 ]);
79 const name = tarballName(pkg.packageJson.name, pkg.packageJson.version);
80 const packed = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(PackedManifest))(
81 yield* exec('tar', ['-xOf', path.join(directory, name), 'package/package.json'])
82 );
83 yield* ensure(
84 packed.name === pkg.packageJson.name && packed.version === pkg.packageJson.version,
85 'Packed npm metadata differs from the approved source'
86 );
87 }
88 });
89
90 const PYTHON_DISTRIBUTION = /\.whl$|\.tar\.gz$/;
91
92 /** Builds the synchronized Python train with the locked release toolchain and keeps its distributions. */
93 export const buildPython = (cwd: string, directory: string) =>
94 Effect.gen(function* () {
95 const fs = yield* FileSystem.FileSystem;
96 const path = yield* Path.Path;
97 const environment = { UV_PROJECT_ENVIRONMENT: path.resolve(cwd, 'python/.venv') };
98 const exec = inDirectory(cwd, environment);
99 yield* exec('uv', ['sync', '--locked', '--only-group', 'release']);
100 yield* inDirectory(path.join(cwd, 'python'), environment)('make', [
101 'build',
102 'BUILD_ARGS=--no-isolation',
103 ]);
104 yield* exec('uv', [
105 'run',
106 '--locked',
107 '--only-group',
108 'release',
109 'python',
110 'scripts/sdk-release/validate-python-candidates.py',
111 ]);
112 const dist = path.resolve(cwd, 'python/dist');
113 for (const name of (yield* fs.readDirectory(dist)).filter(name =>
114 PYTHON_DISTRIBUTION.test(name)
115 ))
116 yield* fs.copyFile(path.join(dist, name), path.join(directory, name));
117 });
118
119 /** Lane-specific source checks and builds for a fresh candidate. */
120 const LANE_BUILDS: Record<
121 Language,
122 {
123 readonly verifyTrain: (
124 cwd: string,
125 selected: ReadonlyArray<SdkRelease>
126 ) => Effect.Effect<void, unknown, ReleaseServices>;
127 readonly build: (
128 cwd: string,
129 directory: string,
130 selected: ReadonlyArray<SdkRelease>
131 ) => Effect.Effect<void, unknown, ReleaseServices>;
132 }
133 > = {
134 typescript: {
135 verifyTrain: () => Effect.void,
136 build: (cwd, directory, selected) =>
137 buildTypescript(cwd, directory, new Set(selected.map(release => release.name))),
138 },
139 python: {
140 verifyTrain: (cwd, selected) =>
141 Effect.gen(function* () {
142 const [release] = selected;
143 const python = yield* discoverPythonPackages(cwd);
144 yield* ensure(
145 python.every(pkg => pkg.version === REGISTRY_VERSION.python(release.newVersion)),
146 'Python candidate train differs from the approved target'
147 );
148 }),
149 build: (cwd, directory) => buildPython(cwd, directory),
150 },
151 };
152
153 const BuildEnvironment = Config.all({
154 runId: Config.schema(PositiveIntFromString, 'GITHUB_RUN_ID'),
155 /** The release PR head; on pull_request, GITHUB_SHA is the merge commit instead. */
156 sha: Config.schema(Sha, 'SDK_SOURCE_SHA'),
157 ref: Config.String('GITHUB_REF'),
158 controllerToken: Config.option(Config.String('SDK_RELEASE_TOKEN')),
159 });
160 type BuildEnvironment = Config.Success<typeof BuildEnvironment>;
161
162 /** Release-PR runs build candidates; source-tag runs publish the approved release-PR bytes. */
163 const isSourceTag = (ref: string) => ref.startsWith('refs/tags/sdk-release/');
164
165 /** The pull_request-only workflow that builds release-PR candidates. */
166 const CANDIDATES_WORKFLOW = 'sdk.release.candidates.yml';
167 const CANDIDATES_WORKFLOW_PATH = `.github/workflows/${CANDIDATES_WORKFLOW}`;
168
169 const WorkflowRuns = Schema.Struct({ workflow_runs: Schema.Array(ReleaseRun) });
170
171 /** GitHub reports step and artifact times to the second, so an artifact can land on either boundary second. */
172 const TIMESTAMP_SLACK_MS = 1_000;
173
174 /** The release-PR build step that uploads the candidate artifact. */
175 const UPLOAD_STEP = 'Retain candidate bytes for publication';
176
177 /**
178 * Requires `artifact` to have been created by the upload step of the newest successful `jobName`
179 * job of `runId`. Every other job and step in that run holds its runtime token and could replace
180 * the artifact. Jobs from every attempt are read because a partial re-run leaves earlier jobs out
181 * of the latest attempt.
182 */
183 const requireCreatedDuring = (
184 runId: number,
185 jobName: string,
186 artifact: CandidateArtifact,
187 where: string
188 ) =>
189 Effect.gen(function* () {
190 const [step] = (yield* jobs(String(runId), 'all'))
191 .filter(job => job.name === jobName && job.conclusion === 'success')
192 .flatMap(job => job.steps)
193 .flatMap(step =>
194 step.name === UPLOAD_STEP &&
195 step.conclusion === 'success' &&
196 step.started_at &&
197 step.completed_at
198 ? [{ startedAt: step.started_at, completedAt: step.completed_at }]
199 : []
200 )
201 .toSorted((a, b) => Date.parse(b.completedAt) - Date.parse(a.completedAt));
202 if (!step)
203 return yield* fail(
204 `The ${jobName} job did not upload a candidate in ${where}; add changesets and request a new release`
205 );
206 const created = Date.parse(artifact.created_at);
207 yield* ensure(
208 Date.parse(step.startedAt) - TIMESTAMP_SLACK_MS <= created &&
209 created <= Date.parse(step.completedAt) + TIMESTAMP_SLACK_MS,
210 `${artifact.name} was created at ${artifact.created_at}, outside its upload step, which ran from ${step.startedAt} to ${step.completedAt}; add changesets and request a new release`
211 );
212 });
213
214 /**
215 * Copies the candidate a release-PR run built and checked on this exact head into `directory`.
216 * Every successful release-PR run on the head passed the candidate checks the merge requires;
217 * the newest one is used, and its artifact must come from that run's build job.
218 */
219 export const adoptApprovedCandidate = (sha: string, language: Language, directory: string) =>
220 Effect.gen(function* () {
221 const { workflow_runs } = yield* request(
222 'GET',
223 `${REPOSITORY_PATH}/actions/workflows/${CANDIDATES_WORKFLOW}/runs?event=pull_request&head_sha=${sha}&status=success&per_page=100`,
224 WorkflowRuns
225 );
226 const [run] = workflow_runs
227 .filter(
228 candidate =>
229 candidate.path === CANDIDATES_WORKFLOW_PATH &&
230 candidate.repository.full_name === SDK_RELEASE_REPOSITORY &&
231 candidate.event === 'pull_request' &&
232 candidate.head_sha === sha &&
233 candidate.conclusion === 'success'
234 )
235 .toSorted((a, b) => b.id - a.id);
236 if (!run)
237 return yield* fail(
238 `No successful release-PR run built the approved ${language} candidate; add changesets and request a new release`
239 );
240 const artifact = yield* reuseCandidate(
241 yield* originalCandidates(run.id),
242 candidateArtifactName(language),
243 run.id,
244 sha
245 );
246 if (Option.isNone(artifact))
247 return yield* fail(
248 `Approved ${language} candidate was deleted from ${run.html_url}; add changesets and request a new release`
249 );
250 yield* requireCreatedDuring(
251 run.id,
252 `SDK candidate (${language})`,
253 artifact.value,
254 run.html_url
255 );
256 yield* downloadCandidate(artifact.value, directory);
257 const manifestPath = `${directory}/${CANDIDATE_MANIFEST}`;
258 const manifest = yield* readJson(CandidateManifest, manifestPath);
259 yield* ensure(
260 manifest.sourceSha === sha &&
261 manifest.producerRun === run.id &&
262 manifest.language === language,
263 'Approved candidate manifest producer/source mismatch'
264 );
265 yield* verifyCandidateFiles(yield* readCandidateFiles(directory), manifest.files);
266 yield* (yield* FileSystem.FileSystem).remove(manifestPath);
267 return {
268 run: run.id,
269 artifactId: artifact.value.id,
270 digest: artifact.value.digest,
271 } satisfies ApprovedCandidate;
272 });
273
274 /**
275 * Makes the checkout carry the versions the candidate publishes. A stable release PR already
276 * commits them; a beta PR commits only its plan, so its frozen plan is applied here, in the
277 * credential-free build and smoke jobs, and never reaches `next`.
278 */
279 export const prepareCandidateSource = (cwd: string, batch: ReleaseBatch) =>
280 channelOf(batch).commitsVersions ? Effect.void : applySdkReleasePlan(cwd, batch.plan);
281
282 /** Builds a release-PR candidate. */
283 const buildCandidate = (
284 env: BuildEnvironment,
285 language: Language,
286 selected: ReadonlyArray<SdkRelease>
287 ) =>
288 Effect.gen(function* () {
289 const fs = yield* FileSystem.FileSystem;
290 const path = yield* Path.Path;
291 const cwd = path.resolve('.');
292 const directory = path.resolve('sdk-candidate', language);
293 yield* fs.makeDirectory(directory, { recursive: true });
294 const lane = LANE_BUILDS[language];
295 yield* lane.verifyTrain(cwd, selected);
296 const { packages } = yield* fromPromise(() => getPackages(cwd));
297 for (const release of selected) {
298 const pkg = packages.find(candidate => candidate.packageJson.name === release.name);
299 yield* ensure(
300 pkg?.packageJson.version === release.newVersion,
301 `Candidate version differs from approved target: ${release.name}`
302 );
303 }
304 yield* lane.build(cwd, directory, selected);
305 const files = yield* Effect.forEach(yield* fs.readDirectory(directory), name =>
306 Effect.gen(function* () {
307 const bytes = yield* fs.readFile(`${directory}/${name}`);
308 return { name, sha256: yield* sha256Hex(bytes), size: bytes.length };
309 })
310 );
311 const manifest = yield* Schema.decodeUnknownEffect(CandidateManifest)({
312 sourceSha: env.sha,
313 producerRun: env.runId,
314 language,
315 files,
316 });
317 yield* fs.writeFileString(`${directory}/${CANDIDATE_MANIFEST}`, formatJson(manifest));
318 });
319
320 const buildMode = (language: Language) =>
321 Effect.gen(function* () {
322 const env = yield* BuildEnvironment;
323 yield* ensure(
324 !Option.exists(env.controllerToken, token => token !== ''),
325 'Candidate jobs must not receive the controller credential'
326 );
327 yield* ensure(
328 !isSourceTag(env.ref),
329 'Source-tag runs publish approved candidates, never builds'
330 );
331 const batch = yield* readJson(ReleaseBatch, BATCH_PATH);
332 const selected = batch.plan.releases.filter(release => release.language === language);
333 yield* ensure(selected.length > 0, `This release plans no ${language} packages`);
334 yield* prepareCandidateSource((yield* Path.Path).resolve('.'), batch);
335 yield* buildCandidate(env, language, selected);
336 });
337
338 const ReportEnvironment = Config.all({
339 artifactId: Config.schema(PositiveIntFromString, 'ARTIFACT_ID'),
340 artifactDigest: Config.String('ARTIFACT_DIGEST'),
341 sourceSha: Config.schema(Sha, 'SOURCE_SHA'),
342 runId: Config.schema(PositiveIntFromString, 'GITHUB_RUN_ID'),
343 summaryPath: Config.String('GITHUB_STEP_SUMMARY'),
344 token: Config.schema(Schema.NonEmptyString, 'SDK_READ_TOKEN'),
345 });
346
347 /** Confirms the uploaded artifact is this run's candidate and records its identity. */
348 const reportMode = (language: Language) =>
349 Effect.gen(function* () {
350 const env = yield* ReportEnvironment;
351 const artifact = yield* request(
352 'GET',
353 `${REPOSITORY_PATH}/actions/artifacts/${env.artifactId}`,
354 ActionArtifact
355 ).pipe(Effect.provide(GitHub.layer(env.token)));
356 yield* reuseCandidate([artifact], candidateArtifactName(language), env.runId, env.sourceSha);
357 yield* ensure(
358 artifact.digest === (yield* normalizeArtifactDigest(env.artifactDigest)),
359 'Uploaded artifact digest differs'
360 );
361 const manifest = yield* readJson(
362 CandidateManifest,
363 `sdk-candidate/${language}/${CANDIDATE_MANIFEST}`
364 );
365 const files = manifest.files
366 .map(file => `- ${file.name}: ${file.sha256} (${file.size} bytes)`)
367 .join('\n');
368 yield* appendFile(
369 env.summaryPath,
370 `${language} candidate ID=${artifact.id} digest=${artifact.digest} producer=${env.runId} source=${env.sourceSha} expires=${artifact.expires_at}\n\n${files}\n`
371 );
372 });
373
374 const ApproveEnvironment = Config.all({
375 sha: Config.schema(Sha, 'GITHUB_SHA'),
376 appSlug: Config.String('SDK_RELEASE_APP_SLUG'),
377 ref: Config.String('GITHUB_REF'),
378 outputPath: Config.String('GITHUB_OUTPUT'),
379 summaryPath: Config.String('GITHUB_STEP_SUMMARY'),
380 token: Config.schema(Schema.NonEmptyString, 'SDK_READ_TOKEN'),
381 controllerToken: Config.option(Config.String('SDK_RELEASE_TOKEN')),
382 });
383
384 /**
385 * Checks the approved release-PR candidate of every selected language once and pins its artifact
386 * ID and digest as a job output. Publish jobs download exactly that artifact, and a retried job
387 * reuses the output, so every attempt publishes the same bytes.
388 */
389 const approveMode = Effect.gen(function* () {
390 const env = yield* ApproveEnvironment;
391 yield* ensure(
392 !Option.exists(env.controllerToken, token => token !== ''),
393 'Candidate jobs must not receive the controller credential'
394 );
395 const path = yield* Path.Path;
396 const batch = yield* readJson(ReleaseBatch, BATCH_PATH);
397 yield* ensure(
398 env.ref === `refs/tags/${releaseTag(batch.id)}`,
399 `Approved candidates are pinned only by the release tag of batch ${batch.id}`
400 );
401 // Written first, so the publish status still reaches the request issue if adoption fails. Only
402 // a plan the release App committed may name that issue: the tag ruleset should stop anyone
403 // else from pushing a release tag, and this keeps a forged one from steering the bot.
404 const trusted = yield* isReleaseAppCommit(env.sha, batch, env.appSlug).pipe(
405 Effect.provide(GitHub.layer(env.token))
406 );
407 if (trusted) yield* appendFile(env.outputPath, `issue=${batch.issue}\n`);
408 const approved: Record<string, ApprovedCandidate> = {};
409 yield* Effect.gen(function* () {
410 for (const language of batchLanguages(batch)) {
411 const candidate = yield* adoptApprovedCandidate(
412 env.sha,
413 language,
414 path.resolve('sdk-candidate', language)
415 );
416 approved[language] = candidate;
417 yield* appendFile(
418 env.summaryPath,
419 `${language}: approved release-PR run ${candidate.run}, artifact ${candidate.artifactId} (${candidate.digest})\n`
420 );
421 }
422 }).pipe(Effect.provide(GitHub.layer(env.token)));
423 const pinned = yield* Schema.decodeUnknownEffect(ApprovedCandidates)(approved);
424 yield* appendFile(
425 env.outputPath,
426 `approved=${JSON.stringify(pinned)}\ntypescript=${'typescript' in pinned}\npython=${'python' in pinned}\n`
427 );
428 });
429
430 /**
431 * `candidates.ts <language>` builds a release-PR candidate, `candidates.ts report <language>`
432 * records its identity, and `candidates.ts approve` pins the approved candidates in a tag run.
433 */
434 const Arguments = Schema.Union([
435 Schema.Tuple([Schema.Literal('approve')]),
436 Schema.Tuple([Language]),
437 Schema.Tuple([Schema.Literal('report'), Language]),
438 ]);
439
440 const main = Effect.gen(function* () {
441 const args = yield* Schema.decodeUnknownEffect(Arguments)(yield* (yield* Stdio.Stdio).args);
442 if (args[0] === 'approve') return yield* approveMode;
443 return yield* args.length === 1 ? buildMode(args[0]) : reportMode(args[1]);
444 });
445
446 if (import.meta.main) runMain(main);