@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 530 lines 18.9 KB
Raw
1 import { dirname, resolve } from 'node:path';
2 import * as BunFileSystem from '@effect/platform-bun/BunFileSystem';
3 import * as BunRuntime from '@effect/platform-bun/BunRuntime';
4 import { Data, Effect, FileSystem, Schema } from 'effect';
5 import { compare, valid } from 'semver';
6
7 export const branch = 'chore/update-composio-client';
8 export const paths = [
9 'pnpm-workspace.yaml',
10 'pnpm-lock.yaml',
11 'python/pyproject.toml',
12 'python/setup.py',
13 'uv.lock',
14 '.changeset/composio-client-update.md',
15 'ts/packages/cli/CHANGELOG.md',
16 ] as const;
17
18 // The pattern admits only X.Y.Z; semver round-trips it, so whitespace and
19 // out-of-range components are rejected too.
20 export const stableVersion = Schema.String.check(
21 Schema.makeFilter(
22 value => /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/.test(value) && valid(value) === value,
23 { message: 'Expected a canonical stable semver version' }
24 )
25 );
26 const sha = Schema.String.check(Schema.isPattern(/^[a-f0-9]{40}$/));
27 const dispatch = Schema.Struct({
28 version: stableVersion,
29 source_sha: sha,
30 source_repository: Schema.Literal('ComposioHQ/composio-client'),
31 });
32 const dispatchEvent = Schema.Struct({ client_payload: Schema.Unknown });
33 const pypiFile = Schema.Struct({ yanked: Schema.Boolean, filename: Schema.String });
34 const npmIndex = Schema.Struct({ versions: Schema.Record(Schema.String, Schema.Unknown) });
35 const pypiIndex = Schema.Struct({
36 releases: Schema.Record(Schema.String, Schema.Array(pypiFile)),
37 });
38 const npmRelease = Schema.Struct({
39 name: Schema.Literal('@composio/client'),
40 version: stableVersion,
41 dist: Schema.Struct({
42 integrity: Schema.NonEmptyString,
43 tarball: Schema.String.check(Schema.isPattern(/^https:\/\/registry\.npmjs\.org\/\S+$/)),
44 }),
45 });
46 const pypiRelease = Schema.Struct({
47 info: Schema.Struct({ name: Schema.Literal('composio-client'), version: stableVersion }),
48 urls: Schema.Array(pypiFile),
49 });
50 const artifactSchema = Schema.Struct({
51 version: stableVersion,
52 base_sha: sha,
53 files: Schema.Array(
54 Schema.Struct({ path: Schema.Literals(paths), content: Schema.String })
55 ).check(Schema.isMinLength(1), Schema.isMaxLength(paths.length)),
56 });
57
58 const decode = <S extends Schema.Codec<unknown, unknown>>(schema: S) =>
59 Schema.decodeUnknownSync(schema);
60 // Inputs crossing a trust boundary must not carry fields the updater ignores.
61 const decodeExact = <S extends Schema.Codec<unknown, unknown>>(schema: S) =>
62 Schema.decodeUnknownSync(schema, { onExcessProperty: 'error' });
63
64 export const isStableVersion = Schema.is(stableVersion);
65
66 export function validateEvent(name: string, payload: unknown) {
67 if (name === 'repository_dispatch') {
68 return decodeExact(dispatch)(decode(dispatchEvent)(payload).client_payload);
69 }
70 if (name !== 'schedule' && name !== 'workflow_dispatch')
71 throw new Error(`Unsupported event: ${name}`);
72 return undefined;
73 }
74
75 export function commonStableVersion(npmData: unknown, pypiData: unknown): string {
76 const npm = decode(npmIndex)(npmData);
77 const pypi = decode(pypiIndex)(pypiData);
78 const versions = Object.keys(npm.versions).filter(
79 version => isStableVersion(version) && pypi.releases[version]?.some(file => !file.yanked)
80 );
81 versions.sort(compare);
82 const latest = versions.at(-1);
83 if (!latest) throw new Error('npm and PyPI have no common stable composio-client release');
84 return latest;
85 }
86
87 export function verifyReleases(version: string, npmData: unknown, pypiData: unknown) {
88 const npm = decode(npmRelease)(npmData);
89 const pypi = decode(pypiRelease)(pypiData);
90 if (
91 npm.version !== version ||
92 pypi.info.version !== version ||
93 !pypi.urls.some(file => !file.yanked && file.filename.endsWith('.whl'))
94 ) {
95 throw new Error(`Both registries must expose usable composio-client ${version} distributions`);
96 }
97 }
98
99 export const decodeArtifact = decodeExact(artifactSchema);
100
101 const catalogPattern = /^(\s*'@composio\/client':\s*)([^\s#]+)(\s*)$/gm;
102 const pythonPattern = /("composio-client==)([^"\s]+)(")/g;
103
104 function onePin(content: string, pattern: RegExp, label: string): string {
105 const matches = [...content.matchAll(pattern)];
106 if (matches.length !== 1) throw new Error(`Expected exactly one ${label} pin`);
107 return decode(stableVersion)(matches[0]?.[2]);
108 }
109
110 function required(files: Record<string, string>, path: string): string {
111 const content = files[path];
112 if (content === undefined) throw new Error(`Missing ${path}`);
113 return content;
114 }
115
116 export function catalogVersion(content: string) {
117 return onePin(content, catalogPattern, 'catalog');
118 }
119
120 export function planUpdates(
121 files: Record<string, string>,
122 targetInput: string,
123 existingCandidate?: string
124 ) {
125 const target = decode(stableVersion)(targetInput);
126 const current = [
127 catalogVersion(required(files, 'pnpm-workspace.yaml')),
128 onePin(required(files, 'python/pyproject.toml'), pythonPattern, 'Python pyproject'),
129 onePin(required(files, 'python/setup.py'), pythonPattern, 'Python setup'),
130 ];
131 if (current[1] !== current[2])
132 throw new Error('Python client pins disagree; reconcile metadata first');
133 if (
134 [...current, ...(existingCandidate ? [decode(stableVersion)(existingCandidate)] : [])].some(
135 version => compare(version, target) > 0
136 )
137 ) {
138 throw new Error(`Refusing to downgrade a current or pending client pin to ${target}`);
139 }
140 const changed: Record<string, string> = {};
141 const releases: string[] = [];
142 if (current[0] !== target) {
143 changed['pnpm-workspace.yaml'] = required(files, 'pnpm-workspace.yaml').replace(
144 catalogPattern,
145 `$1${target}$3`
146 );
147 releases.push("'@composio/core': patch");
148 const changelog = required(files, 'ts/packages/cli/CHANGELOG.md');
149 const header = '# @composio/cli\n\n## Unreleased\n';
150 const marker = '### Patch Changes\n';
151 if (!changelog.startsWith(header)) {
152 throw new Error('CLI changelog is missing its Unreleased section');
153 }
154 const nextRelease = changelog.indexOf('\n## ', header.length);
155 const end = nextRelease === -1 ? changelog.length : nextRelease;
156 const unreleased = changelog.slice(header.length, end);
157 const note = `- Update @composio/client to ${target} to include the latest API changes.\n`;
158 const updated = unreleased.includes(marker)
159 ? unreleased.replace(marker, `${marker}\n${note}`)
160 : `\n${marker}\n${note}${unreleased}`;
161 changed['ts/packages/cli/CHANGELOG.md'] = `${header}${updated}${changelog.slice(end)}`;
162 }
163 if (current[1] !== target) {
164 releases.push("'@composio/python-sdk': patch");
165 for (const path of ['python/pyproject.toml', 'python/setup.py']) {
166 changed[path] = required(files, path).replace(pythonPattern, `$1${target}$3`);
167 }
168 }
169 if (releases.length > 0) {
170 changed['.changeset/composio-client-update.md'] =
171 `---\n${releases.join('\n')}\n---\n\nUpdate the API client dependencies to ${target} to include the latest API changes.\n`;
172 }
173 return { changed, typescript: current[0] !== target, python: current[1] !== target };
174 }
175
176 export class ClientUpdateIoError extends Data.TaggedError('ClientUpdateIoError')<{
177 readonly operation: string;
178 readonly path: string;
179 readonly message: string;
180 readonly cause: unknown;
181 }> {}
182 export class ClientUpdateRegistryError extends Data.TaggedError('ClientUpdateRegistryError')<{
183 readonly url: string;
184 readonly message: string;
185 readonly cause?: unknown;
186 }> {}
187 export class ClientUpdateCommandError extends Data.TaggedError('ClientUpdateCommandError')<{
188 readonly command: readonly string[];
189 readonly message: string;
190 readonly cause?: unknown;
191 }> {}
192 export class ClientUpdateValidationError extends Data.TaggedError('ClientUpdateValidationError')<{
193 readonly message: string;
194 readonly cause?: unknown;
195 }> {}
196 type ClientUpdateError =
197 | ClientUpdateIoError
198 | ClientUpdateRegistryError
199 | ClientUpdateCommandError
200 | ClientUpdateValidationError;
201
202 const checked = <A>(operation: () => A) =>
203 Effect.try({
204 try: operation,
205 catch: cause =>
206 new ClientUpdateValidationError({
207 message: cause instanceof Error ? cause.message : 'Client update validation failed',
208 cause,
209 }),
210 });
211 const file = <A, E>(
212 operation: string,
213 path: string,
214 action: (fs: FileSystem.FileSystem) => Effect.Effect<A, E>
215 ) =>
216 Effect.gen(function* () {
217 return yield* action(yield* FileSystem.FileSystem);
218 }).pipe(
219 Effect.mapError(
220 cause =>
221 new ClientUpdateIoError({ operation, path, message: `Cannot ${operation}: ${path}`, cause })
222 )
223 );
224 const read = (path: string) => file('read file', path, fs => fs.readFileString(path));
225 const write = (path: string, content: string) =>
226 file('write file', path, fs => fs.writeFileString(path, content));
227 const directory = (path: string) =>
228 file('create directory', path, fs => fs.makeDirectory(path, { recursive: true }));
229
230 const registryJson = (url: string) =>
231 Effect.gen(function* () {
232 const response = yield* Effect.tryPromise({
233 try: signal =>
234 fetch(url, {
235 signal: AbortSignal.any([signal, AbortSignal.timeout(60_000)]),
236 headers: { accept: 'application/json' },
237 redirect: 'error',
238 }),
239 catch: cause =>
240 new ClientUpdateRegistryError({ url, message: `Registry request failed: ${url}`, cause }),
241 });
242 if (!response.ok)
243 return yield* Effect.fail(
244 new ClientUpdateRegistryError({
245 url,
246 message: `Registry request failed: ${url} (${response.status})`,
247 })
248 );
249 return yield* Effect.tryPromise({
250 try: () => response.json(),
251 catch: cause =>
252 new ClientUpdateRegistryError({ url, message: `Invalid registry JSON: ${url}`, cause }),
253 });
254 });
255
256 const processResult = (args: string[], cwd: string, capture = false) =>
257 Effect.acquireUseRelease(
258 Effect.try({
259 try: () =>
260 Bun.spawn(args, {
261 cwd,
262 stdout: capture ? 'pipe' : 'inherit',
263 stderr: capture ? 'ignore' : 'inherit',
264 }),
265 catch: cause =>
266 new ClientUpdateCommandError({
267 command: args,
268 message: `Cannot start command: ${args.join(' ')}`,
269 cause,
270 }),
271 }),
272 child =>
273 Effect.gen(function* () {
274 const [stdout, exitCode] = yield* Effect.all(
275 [
276 capture
277 ? Effect.tryPromise({
278 try: () => new Response(child.stdout).text(),
279 catch: cause =>
280 new ClientUpdateCommandError({
281 command: args,
282 message: `Cannot read command output: ${args.join(' ')}`,
283 cause,
284 }),
285 })
286 : Effect.succeed(''),
287 Effect.tryPromise({
288 try: () => child.exited,
289 catch: cause =>
290 new ClientUpdateCommandError({
291 command: args,
292 message: `Cannot await command: ${args.join(' ')}`,
293 cause,
294 }),
295 }),
296 ],
297 { concurrency: 'unbounded' }
298 );
299 return { stdout, exitCode };
300 }),
301 child =>
302 Effect.try({
303 try: () => {
304 if (child.exitCode === null) child.kill();
305 },
306 catch: cause =>
307 new ClientUpdateCommandError({
308 command: args,
309 message: 'Cannot stop interrupted command',
310 cause,
311 }),
312 }).pipe(Effect.ignore)
313 );
314 const command = (args: string[], cwd: string) =>
315 Effect.gen(function* () {
316 const result = yield* processResult(args, cwd);
317 if (result.exitCode !== 0)
318 return yield* Effect.fail(
319 new ClientUpdateCommandError({
320 command: args,
321 message: `Command failed: ${args.join(' ')}`,
322 })
323 );
324 });
325 const git = (args: string[], root: string, allowFailure = false) =>
326 Effect.gen(function* () {
327 const result = yield* processResult(['git', ...args], root, true);
328 if (!allowFailure && result.exitCode !== 0)
329 return yield* Effect.fail(
330 new ClientUpdateCommandError({
331 command: ['git', ...args],
332 message: `Git command failed: ${args[0]}`,
333 })
334 );
335 return result;
336 });
337 const output = (name: string, value: string) =>
338 Effect.gen(function* () {
339 const path = process.env.GITHUB_OUTPUT;
340 if (path)
341 yield* file('append workflow output', path, fs =>
342 fs.writeFileString(path, `${name}=${value}\n`, { flag: 'a' })
343 );
344 });
345
346 export const prepare = (
347 root: string,
348 options: {
349 eventName: string;
350 event: unknown;
351 registry?: (url: string) => Effect.Effect<unknown, ClientUpdateError>;
352 runCommand?: (
353 args: string[],
354 cwd: string
355 ) => Effect.Effect<void, ClientUpdateError, FileSystem.FileSystem>;
356 setOutput?: (
357 name: string,
358 value: string
359 ) => Effect.Effect<void, ClientUpdateError, FileSystem.FileSystem>;
360 }
361 ) =>
362 Effect.gen(function* () {
363 const readRegistry = options.registry ?? registryJson;
364 const runCommand = options.runCommand ?? command;
365 const setOutput = options.setOutput ?? output;
366 const source = yield* checked(() => validateEvent(options.eventName, options.event));
367 const artifactPath = resolve(root, '.artifacts/client-update/update.json');
368 yield* file('remove stale artifact', artifactPath, fs =>
369 fs.remove(artifactPath, { force: true })
370 );
371 const [npm, pypi] = yield* Effect.all(
372 [
373 readRegistry('https://registry.npmjs.org/@composio%2fclient'),
374 readRegistry('https://pypi.org/pypi/composio-client/json'),
375 ],
376 { concurrency: 'unbounded' }
377 );
378 const version = yield* checked(() => commonStableVersion(npm, pypi));
379 // A dispatch may race the registry CDN. Retry next hour rather than silently acknowledging an older release.
380 if (source && compare(version, source.version) < 0)
381 return yield* Effect.fail(
382 new ClientUpdateValidationError({
383 message: `Dispatched ${source.version} is not yet available on both registries`,
384 })
385 );
386 const [npmExact, pypiExact] = yield* Effect.all(
387 [
388 readRegistry(`https://registry.npmjs.org/@composio%2fclient/${version}`),
389 readRegistry(`https://pypi.org/pypi/composio-client/${version}/json`),
390 ],
391 { concurrency: 'unbounded' }
392 );
393 yield* checked(() => verifyReleases(version, npmExact, pypiExact));
394 const files = Object.fromEntries(
395 yield* Effect.all(
396 [
397 'pnpm-workspace.yaml',
398 'python/pyproject.toml',
399 'python/setup.py',
400 'ts/packages/cli/CHANGELOG.md',
401 ].map(path =>
402 Effect.gen(function* () {
403 return [path, yield* read(resolve(root, path))];
404 })
405 ),
406 { concurrency: 'unbounded' }
407 )
408 );
409 const candidate = yield* git(
410 ['show', `refs/remotes/origin/${branch}:pnpm-workspace.yaml`],
411 root,
412 true
413 );
414 const existing =
415 candidate.exitCode === 0 ? yield* checked(() => catalogVersion(candidate.stdout)) : undefined;
416 const plan = yield* checked(() => planUpdates(files, version, existing));
417 const changed = Object.keys(plan.changed).length > 0;
418 yield* setOutput('changed', String(changed));
419 yield* setOutput('version', version);
420 if (!changed) {
421 yield* Effect.sync(() => console.log(`All client pins already use ${version}`));
422 return;
423 }
424 for (const [path, content] of Object.entries(plan.changed)) {
425 yield* directory(dirname(resolve(root, path)));
426 yield* write(resolve(root, path), content);
427 }
428 if (plan.typescript) {
429 yield* runCommand(
430 ['pnpm', 'install', '--lockfile-only', '--ignore-scripts', '--no-frozen-lockfile'],
431 root
432 );
433 }
434 if (plan.python) {
435 const python = yield* checked(() => decode(Schema.NonEmptyString)(Bun.which('python')));
436 yield* runCommand(
437 ['uv', 'lock', '--python', python, '--upgrade-package', 'composio-client'],
438 root
439 );
440 }
441 const diff = yield* git(['diff', '--name-only'], root);
442 const diffPaths = diff.stdout.trim().split('\n').filter(Boolean);
443 if (diffPaths.some(path => !paths.some(allowed => allowed === path))) {
444 return yield* Effect.fail(
445 new ClientUpdateValidationError({
446 message: 'Lock generation changed files outside the client update allowlist',
447 })
448 );
449 }
450 const base = yield* git(['rev-parse', 'HEAD'], root);
451 const baseSha = yield* checked(() => decode(sha)(base.stdout.trim()));
452 const artifactFiles = yield* Effect.all(
453 [...new Set([...diffPaths, ...Object.keys(plan.changed)])].map(path =>
454 Effect.gen(function* () {
455 return { path, content: yield* read(resolve(root, path)) };
456 })
457 ),
458 { concurrency: 'unbounded' }
459 );
460 const artifact = yield* checked(() =>
461 decodeArtifact({
462 version,
463 base_sha: baseSha,
464 files: artifactFiles,
465 })
466 );
467 yield* directory(resolve(root, '.artifacts/client-update'));
468 yield* write(artifactPath, JSON.stringify(artifact));
469 yield* Effect.sync(() =>
470 console.log(`Prepared client ${version}; npm and PyPI exact distributions verified`)
471 );
472 });
473
474 export const applyArtifact = (root: string, data: unknown, baseSha: string) =>
475 Effect.gen(function* () {
476 const artifact = yield* checked(() => decodeArtifact(data));
477 if (artifact.base_sha !== baseSha)
478 return yield* Effect.fail(
479 new ClientUpdateValidationError({
480 message: 'next advanced after preparation; retry against its current revision',
481 })
482 );
483 if (new Set(artifact.files.map(file => file.path)).size !== artifact.files.length)
484 return yield* Effect.fail(
485 new ClientUpdateValidationError({ message: 'Duplicate artifact path' })
486 );
487 for (const file of artifact.files) {
488 yield* directory(dirname(resolve(root, file.path)));
489 yield* write(resolve(root, file.path), file.content);
490 }
491 return artifact.version;
492 });
493
494 if (import.meta.main) {
495 const main = Effect.gen(function* () {
496 const root = process.cwd();
497 if (process.argv[2] === 'prepare') {
498 const eventText = process.env.GITHUB_EVENT_PATH
499 ? yield* read(process.env.GITHUB_EVENT_PATH)
500 : '{}';
501 yield* prepare(root, {
502 eventName: process.env.GITHUB_EVENT_NAME ?? 'workflow_dispatch',
503 event: yield* checked(() => JSON.parse(eventText)),
504 });
505 } else if (process.argv[2] === 'apply') {
506 const head = yield* git(['rev-parse', 'HEAD'], root);
507 const artifactText = yield* read(resolve(root, '.artifacts/client-update/update.json'));
508 const version = yield* applyArtifact(
509 root,
510 yield* checked(() => JSON.parse(artifactText)),
511 head.stdout.trim()
512 );
513 yield* output('version', version);
514 } else
515 return yield* Effect.fail(
516 new ClientUpdateValidationError({
517 message: 'Usage: bun scripts/update-composio-client.ts prepare|apply',
518 })
519 );
520 });
521 BunRuntime.runMain(
522 main.pipe(
523 Effect.tapError(error => Effect.sync(() => console.error(error.message))),
524 Effect.provide(BunFileSystem.layer)
525 ),
526 {
527 disableErrorReporting: true,
528 }
529 );
530 }