Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 354 lines 14.1 KB
Raw
1 import { Effect, FileSystem, Path, Schema, Stdio } from 'effect';
2 import { HttpClient } from 'effect/http';
3 import { Toml } from 'effect/encoding';
4 import { getPackages } from '@manypkg/get-packages';
5 import {
6 CANDIDATE_MANIFEST,
7 CandidateManifest,
8 readCandidateFiles,
9 verifyCandidateFiles,
10 } from './artifacts';
11 import { prepareCandidateSource, tarballName } from './candidates';
12 import { NPM_REGISTRY, REGISTRY_VERSION, type Language } from './lanes';
13 import { discoverPythonPackages } from './python-metadata';
14 import {
15 ensure,
16 fail,
17 fromPromise,
18 inDirectory,
19 readJson,
20 ReleaseError,
21 runMain,
22 writeFile,
23 } from './runtime';
24 import { BATCH_PATH, ReleaseBatch } from './source';
25 import {
26 discoverProviderCases,
27 type ProviderCompatibilityCase,
28 } from '../../ts/scripts/check-provider-compatibility';
29
30 const PYPI_INDEX = 'https://pypi.org/simple';
31 const Versioned = Schema.Struct({ version: Schema.String });
32 const MiseTools = Schema.Struct({ tools: Schema.Struct({ python: Schema.String }) });
33
34 type Release = ReleaseBatch['plan']['releases'][number];
35
36 /** Third-party packages come from the registry; SDK packages come from the candidate directory. */
37 export const npmInstallArgs = (
38 candidates: string,
39 releases: ReadonlyArray<Pick<Release, 'name' | 'newVersion'>>,
40 others: ReadonlyArray<string>
41 ) => [
42 'install',
43 '--ignore-scripts',
44 '--no-audit',
45 '--no-fund',
46 `--registry=${NPM_REGISTRY}`,
47 ...others,
48 ...releases.map(release => `${candidates}/${tarballName(release.name, release.newVersion)}`),
49 ];
50
51 /** The candidate wheel file for that package and version. */
52 export const pythonRequirement = (
53 candidates: string,
54 wheels: ReadonlyArray<string>,
55 name: string,
56 version: string
57 ) => {
58 const prefix = `${name.replaceAll('-', '_')}-${version}-`;
59 const wheel = wheels.find(file => file.startsWith(prefix) && file.endsWith('.whl'));
60 return wheel && `${candidates}/${wheel}`;
61 };
62
63 const installedVersion = (manifest: string) =>
64 Effect.map(readJson(Versioned, manifest), value => value.version);
65
66 /** Peers that are not released themselves resolve to the workspace or installed version. */
67 const externalPeers = (
68 npm: ReadonlyArray<Release>,
69 packages: Awaited<ReturnType<typeof getPackages>>['packages']
70 ) =>
71 Effect.gen(function* () {
72 const path = yield* Path.Path;
73 const peers = new Map<string, string>();
74 for (const release of npm) {
75 const pkg = packages.find(candidate => candidate.packageJson.name === release.name);
76 for (const name of Object.keys(pkg?.packageJson.peerDependencies ?? {})) {
77 if (npm.some(selected => selected.name === name)) continue;
78 const local = packages.find(candidate => candidate.packageJson.name === name);
79 const version =
80 local?.packageJson.version ??
81 (yield* installedVersion(
82 path.join(pkg?.dir ?? '', 'node_modules', name, 'package.json')
83 ));
84 yield* ensure(
85 !peers.has(name) || peers.get(name) === version,
86 `Selected public consumers require incompatible versions of ${name}`
87 );
88 peers.set(name, version);
89 }
90 }
91 return peers;
92 });
93
94 const PROVIDER_TOOL_FIXTURE = `
95 import type { Tool, ExecuteToolFn } from '@composio/core';
96 const tool: Tool = { slug: 'RELEASE_CHECK', name: 'Release check', description: 'Checks public provider conversion', inputParameters: { type: 'object', properties: { value: { type: 'string' } }, required: ['value'] }, toolkit: { slug: 'release', name: 'Release' }, tags: [] };
97 const execute: ExecuteToolFn = async () => ({ data: {}, error: null, successful: true });
98 `;
99
100 /** Type-checks and runs a strict consumer of the installed packages and provider contracts. */
101 export const checkStrictConsumer = (
102 directory: string,
103 names: ReadonlyArray<string>,
104 providers: ReadonlyArray<ProviderCompatibilityCase>
105 ) =>
106 Effect.gen(function* () {
107 const path = yield* Path.Path;
108 const command = inDirectory(directory);
109 const imports = names
110 .map(
111 (name, index) =>
112 `import * as sdk${index} from '${name}';\nconsole.log(Object.keys(sdk${index}).length);`
113 )
114 .join('\n');
115 const providerContracts = providers.length
116 ? `${PROVIDER_TOOL_FIXTURE}${providers
117 .map(
118 (provider, index) =>
119 `import { ${provider.providerExportName} as Provider${index} } from '${provider.name}';\nconst wrapped${index} = new Provider${index}().wrapTool(tool${provider.agentic ? ', execute' : ''});\nif (!wrapped${index}) throw new Error('${provider.name} failed public conversion');`
120 )
121 .join('\n')}`
122 : '';
123 yield* writeFile(path.join(directory, 'consumer.ts'), imports + providerContracts);
124 const providerDeclarationsOwnedByProviderCi = providers.length ? ['--skipLibCheck'] : [];
125 yield* command(path.join(directory, 'node_modules/.bin/tsc'), [
126 '--strict',
127 '--noEmit',
128 ...providerDeclarationsOwnedByProviderCi,
129 '--target',
130 'ES2022',
131 '--module',
132 'NodeNext',
133 '--moduleResolution',
134 'NodeNext',
135 'consumer.ts',
136 ]);
137 yield* command('node', ['consumer.ts']);
138 });
139
140 /** Installs the candidate npm packages fresh and type-checks a strict consumer. */
141 const verifyNpmConsumer = (
142 cwd: string,
143 batch: ReleaseBatch,
144 directory: string,
145 candidates: string
146 ) =>
147 Effect.gen(function* () {
148 const path = yield* Path.Path;
149 const command = inDirectory(directory);
150 const npm = batch.plan.releases.filter(release => release.language === 'typescript');
151 const { packages } = yield* fromPromise(() => getPackages(cwd));
152 const peers = yield* externalPeers(npm, packages);
153 const typescript = yield* installedVersion(
154 path.resolve(cwd, 'node_modules/typescript/package.json')
155 );
156 const nodeTypes = yield* installedVersion(
157 path.resolve(cwd, 'node_modules/@types/node/package.json')
158 );
159 yield* writeFile(
160 path.join(directory, 'package.json'),
161 JSON.stringify({ private: true, type: 'module' })
162 );
163 yield* command(
164 'npm',
165 npmInstallArgs(candidates, npm, [
166 `typescript@${typescript}`,
167 `@types/node@${nodeTypes}`,
168 ...[...peers].map(([name, version]) => `${name}@${version}`),
169 ])
170 );
171 for (const release of npm) {
172 const installed = yield* installedVersion(
173 path.join(directory, 'node_modules', release.name, 'package.json')
174 );
175 yield* ensure(
176 installed === release.newVersion &&
177 packages.some(
178 pkg =>
179 pkg.packageJson.name === release.name &&
180 pkg.packageJson.version === release.newVersion
181 ),
182 `Candidate consumer installed the wrong version: ${release.name}`
183 );
184 }
185 const providers = (yield* fromPromise(() => discoverProviderCases())).filter(provider =>
186 npm.some(release => release.name === provider.name)
187 );
188 yield* checkStrictConsumer(
189 directory,
190 npm.map(release => release.name),
191 providers
192 );
193 });
194
195 const pythonVerifier = (packages: ReadonlyArray<string>, version: string) =>
196 `import importlib, importlib.metadata, pathlib\npackages = ${JSON.stringify(packages)}\nfor name in packages:\n distribution = importlib.metadata.distribution(name)\n assert distribution.version == ${JSON.stringify(version)}, (name, distribution.version)\n markers = [file for file in distribution.files or [] if str(file).endswith('/py.typed')]\n assert markers, ('Missing typing metadata', name)\n modules = {str(marker).split('/')[0] for marker in markers}\n for module in modules:\n importlib.import_module(module)\n assert pathlib.Path(distribution.locate_file(module), 'py.typed').exists()\nprint('Verified public imports, exact versions and typing metadata:', packages)\n`;
197
198 const pythonVersion = (cwd: string) =>
199 Effect.gen(function* () {
200 const fs = yield* FileSystem.FileSystem;
201 const path = yield* Path.Path;
202 const mise = yield* Effect.fromResult(
203 Toml.parse(yield* fs.readFileString(path.resolve(cwd, 'mise.toml')))
204 );
205 return (yield* Schema.decodeUnknownEffect(MiseTools)(mise)).tools.python;
206 });
207
208 const candidateWheels = (candidates: string) =>
209 Effect.flatMap(FileSystem.FileSystem, fs => fs.readDirectory(candidates));
210
211 /** Each Python package gets its own environment: providers have incompatible dependency trains. */
212 const verifyPythonConsumer = (
213 cwd: string,
214 batch: ReleaseBatch,
215 directory: string,
216 candidates: string
217 ) =>
218 Effect.gen(function* () {
219 const path = yield* Path.Path;
220 const command = inDirectory(directory);
221 const version = REGISTRY_VERSION.python(
222 batch.plan.releases.find(release => release.language === 'python')?.newVersion ?? ''
223 );
224 const packages = yield* discoverPythonPackages(cwd);
225 const python = yield* pythonVersion(cwd);
226 const wheels = yield* candidateWheels(candidates);
227 const requirement = (name: string) =>
228 Effect.fromNullishOr(pythonRequirement(candidates, wheels, name, version)).pipe(
229 Effect.mapError(() => new ReleaseError({ message: `Missing candidate wheel: ${name}` }))
230 );
231 for (const pkg of packages) {
232 const environment = path.join(directory, pkg.name);
233 yield* command('uv', ['venv', '--python', python, environment]);
234 const executable = path.join(environment, 'bin/python');
235 const names = [...new Set(['composio', pkg.name])];
236 yield* command('uv', [
237 'pip',
238 'install',
239 '--python',
240 executable,
241 '--index-url',
242 PYPI_INDEX,
243 ...(yield* Effect.forEach(names, requirement)),
244 ]);
245 yield* command('uv', ['pip', 'check', '--python', executable]);
246 yield* writeFile(path.join(environment, 'verify.py'), pythonVerifier(names, version));
247 yield* command(executable, [path.join(environment, 'verify.py')]);
248 }
249 });
250
251 const providerImport = (name: string) =>
252 `import importlib, importlib.metadata\ndistribution = importlib.metadata.distribution(${JSON.stringify(name)})\nmodules = {str(file).split('/')[0] for file in distribution.files or [] if str(file).endswith('/__init__.py')}\nassert modules, ('No importable modules', ${JSON.stringify(name)})\nfor module in sorted(modules):\n importlib.import_module(module)\nprint('Imported', ${JSON.stringify(name)}, distribution.version)\n`;
253
254 /**
255 * Providers depend on an unpinned `composio`, so users can upgrade core alone: install the
256 * candidate core wheel with each provider's latest published version and import the provider.
257 */
258 export const verifyPublishedProviders = (
259 cwd: string,
260 candidates: string,
261 version: string,
262 directory: string
263 ) =>
264 Effect.gen(function* () {
265 const path = yield* Path.Path;
266 const command = inDirectory(directory);
267 const python = yield* pythonVersion(cwd);
268 const core = pythonRequirement(
269 candidates,
270 yield* candidateWheels(candidates),
271 'composio',
272 version
273 );
274 if (!core) return yield* fail('Missing candidate wheel: composio');
275 const providers = (yield* discoverPythonPackages(cwd)).filter(pkg => pkg.name !== 'composio');
276 const failures: string[] = [];
277 const client = yield* HttpClient.HttpClient;
278 for (const provider of providers) {
279 // A provider released for the first time has no published version to check against yet.
280 const published = yield* client.get(`https://pypi.org/pypi/${provider.name}/json`);
281 if (published.status === 404) continue;
282 const environment = path.join(directory, `published-${provider.name}`);
283 const executable = path.join(environment, 'bin/python');
284 yield* Effect.gen(function* () {
285 yield* command('uv', ['venv', '--python', python, environment]);
286 yield* command('uv', [
287 'pip',
288 'install',
289 '--python',
290 executable,
291 '--index-url',
292 PYPI_INDEX,
293 core,
294 provider.name,
295 ]);
296 yield* writeFile(path.join(environment, 'verify.py'), providerImport(provider.name));
297 yield* command(executable, [path.join(environment, 'verify.py')]);
298 }).pipe(
299 Effect.catch(error =>
300 Effect.sync(() =>
301 failures.push(
302 `Candidate composio breaks the latest published ${provider.name}: ${error.message}`
303 )
304 )
305 )
306 );
307 }
308 yield* ensure(failures.length === 0, failures.join('\n'));
309 });
310
311 const CONSUMERS: Record<Language, typeof verifyNpmConsumer | typeof verifyPythonConsumer> = {
312 typescript: verifyNpmConsumer,
313 python: verifyPythonConsumer,
314 };
315
316 export const readVerifiedCandidate = (candidates: string) =>
317 Effect.gen(function* () {
318 const manifest = yield* readJson(CandidateManifest, `${candidates}/${CANDIDATE_MANIFEST}`);
319 const files = yield* readCandidateFiles(candidates);
320 yield* verifyCandidateFiles(files, manifest.files);
321 return { manifest, files };
322 });
323
324 /** Checks a downloaded candidate against its manifest, then installs and imports it like a user. */
325 export const verifyCandidateConsumers = (cwd: string, batch: ReleaseBatch, candidates: string) =>
326 Effect.scoped(
327 Effect.gen(function* () {
328 const fs = yield* FileSystem.FileSystem;
329 const { manifest } = yield* readVerifiedCandidate(candidates);
330 const directory = yield* fs.makeTempDirectoryScoped({ prefix: 'sdk-candidate-consumer-' });
331 yield* CONSUMERS[manifest.language](cwd, batch, directory, candidates);
332 const python = batch.plan.releases.find(release => release.language === 'python');
333 if (manifest.language === 'python' && python)
334 yield* verifyPublishedProviders(
335 cwd,
336 candidates,
337 REGISTRY_VERSION.python(python.newVersion),
338 directory
339 );
340 })
341 );
342
343 /** `consumers.ts <candidate-dir>` installs a downloaded candidate like a user would. */
344 const main = Effect.gen(function* () {
345 const path = yield* Path.Path;
346 const [candidates] = yield* Schema.decodeUnknownEffect(Schema.Tuple([Schema.String]))(
347 yield* (yield* Stdio.Stdio).args
348 );
349 const batch = yield* readJson(ReleaseBatch, BATCH_PATH);
350 yield* prepareCandidateSource(path.resolve('.'), batch);
351 yield* verifyCandidateConsumers(path.resolve('.'), batch, path.resolve(candidates));
352 });
353
354 if (import.meta.main) runMain(main);