Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | import { Effect, FileSystem, Path, Schema, Stdio } from 'effect'; |
| 2 | import { HttpClient } from 'effect/http'; |
| 3 | import { Toml } from 'effect/encoding'; |
| 4 | import { getPackages } from '@manypkg/get-packages'; |
| 5 | import { |
| 6 | CANDIDATE_MANIFEST, |
| 7 | CandidateManifest, |
| 8 | readCandidateFiles, |
| 9 | verifyCandidateFiles, |
| 10 | } from './artifacts'; |
| 11 | import { prepareCandidateSource, tarballName } from './candidates'; |
| 12 | import { NPM_REGISTRY, REGISTRY_VERSION, type Language } from './lanes'; |
| 13 | import { discoverPythonPackages } from './python-metadata'; |
| 14 | import { |
| 15 | ensure, |
| 16 | fail, |
| 17 | fromPromise, |
| 18 | inDirectory, |
| 19 | readJson, |
| 20 | ReleaseError, |
| 21 | runMain, |
| 22 | writeFile, |
| 23 | } from './runtime'; |
| 24 | import { BATCH_PATH, ReleaseBatch } from './source'; |
| 25 | import { |
| 26 | discoverProviderCases, |
| 27 | type ProviderCompatibilityCase, |
| 28 | } from '../../ts/scripts/check-provider-compatibility'; |
| 29 | |
| 30 | const PYPI_INDEX = 'https://pypi.org/simple'; |
| 31 | const Versioned = Schema.Struct({ version: Schema.String }); |
| 32 | const MiseTools = Schema.Struct({ tools: Schema.Struct({ python: Schema.String }) }); |
| 33 | |
| 34 | type Release = ReleaseBatch['plan']['releases'][number]; |
| 35 | |
| 36 | /** Third-party packages come from the registry; SDK packages come from the candidate directory. */ |
| 37 | export const npmInstallArgs = ( |
| 38 | candidates: string, |
| 39 | releases: ReadonlyArray<Pick<Release, 'name' | 'newVersion'>>, |
| 40 | others: ReadonlyArray<string> |
| 41 | ) => [ |
| 42 | 'install', |
| 43 | '--ignore-scripts', |
| 44 | '--no-audit', |
| 45 | '--no-fund', |
| 46 | `--registry=${NPM_REGISTRY}`, |
| 47 | ...others, |
| 48 | ...releases.map(release => `${candidates}/${tarballName(release.name, release.newVersion)}`), |
| 49 | ]; |
| 50 | |
| 51 | /** The candidate wheel file for that package and version. */ |
| 52 | export const pythonRequirement = ( |
| 53 | candidates: string, |
| 54 | wheels: ReadonlyArray<string>, |
| 55 | name: string, |
| 56 | version: string |
| 57 | ) => { |
| 58 | const prefix = `${name.replaceAll('-', '_')}-${version}-`; |
| 59 | const wheel = wheels.find(file => file.startsWith(prefix) && file.endsWith('.whl')); |
| 60 | return wheel && `${candidates}/${wheel}`; |
| 61 | }; |
| 62 | |
| 63 | const installedVersion = (manifest: string) => |
| 64 | Effect.map(readJson(Versioned, manifest), value => value.version); |
| 65 | |
| 66 | /** Peers that are not released themselves resolve to the workspace or installed version. */ |
| 67 | const externalPeers = ( |
| 68 | npm: ReadonlyArray<Release>, |
| 69 | packages: Awaited<ReturnType<typeof getPackages>>['packages'] |
| 70 | ) => |
| 71 | Effect.gen(function* () { |
| 72 | const path = yield* Path.Path; |
| 73 | const peers = new Map<string, string>(); |
| 74 | for (const release of npm) { |
| 75 | const pkg = packages.find(candidate => candidate.packageJson.name === release.name); |
| 76 | for (const name of Object.keys(pkg?.packageJson.peerDependencies ?? {})) { |
| 77 | if (npm.some(selected => selected.name === name)) continue; |
| 78 | const local = packages.find(candidate => candidate.packageJson.name === name); |
| 79 | const version = |
| 80 | local?.packageJson.version ?? |
| 81 | (yield* installedVersion( |
| 82 | path.join(pkg?.dir ?? '', 'node_modules', name, 'package.json') |
| 83 | )); |
| 84 | yield* ensure( |
| 85 | !peers.has(name) || peers.get(name) === version, |
| 86 | `Selected public consumers require incompatible versions of ${name}` |
| 87 | ); |
| 88 | peers.set(name, version); |
| 89 | } |
| 90 | } |
| 91 | return peers; |
| 92 | }); |
| 93 | |
| 94 | const PROVIDER_TOOL_FIXTURE = ` |
| 95 | import type { Tool, ExecuteToolFn } from '@composio/core'; |
| 96 | const tool: Tool = { slug: 'RELEASE_CHECK', name: 'Release check', description: 'Checks public provider conversion', inputParameters: { type: 'object', properties: { value: { type: 'string' } }, required: ['value'] }, toolkit: { slug: 'release', name: 'Release' }, tags: [] }; |
| 97 | const execute: ExecuteToolFn = async () => ({ data: {}, error: null, successful: true }); |
| 98 | `; |
| 99 | |
| 100 | /** Type-checks and runs a strict consumer of the installed packages and provider contracts. */ |
| 101 | export const checkStrictConsumer = ( |
| 102 | directory: string, |
| 103 | names: ReadonlyArray<string>, |
| 104 | providers: ReadonlyArray<ProviderCompatibilityCase> |
| 105 | ) => |
| 106 | Effect.gen(function* () { |
| 107 | const path = yield* Path.Path; |
| 108 | const command = inDirectory(directory); |
| 109 | const imports = names |
| 110 | .map( |
| 111 | (name, index) => |
| 112 | `import * as sdk${index} from '${name}';\nconsole.log(Object.keys(sdk${index}).length);` |
| 113 | ) |
| 114 | .join('\n'); |
| 115 | const providerContracts = providers.length |
| 116 | ? `${PROVIDER_TOOL_FIXTURE}${providers |
| 117 | .map( |
| 118 | (provider, index) => |
| 119 | `import { ${provider.providerExportName} as Provider${index} } from '${provider.name}';\nconst wrapped${index} = new Provider${index}().wrapTool(tool${provider.agentic ? ', execute' : ''});\nif (!wrapped${index}) throw new Error('${provider.name} failed public conversion');` |
| 120 | ) |
| 121 | .join('\n')}` |
| 122 | : ''; |
| 123 | yield* writeFile(path.join(directory, 'consumer.ts'), imports + providerContracts); |
| 124 | const providerDeclarationsOwnedByProviderCi = providers.length ? ['--skipLibCheck'] : []; |
| 125 | yield* command(path.join(directory, 'node_modules/.bin/tsc'), [ |
| 126 | '--strict', |
| 127 | '--noEmit', |
| 128 | ...providerDeclarationsOwnedByProviderCi, |
| 129 | '--target', |
| 130 | 'ES2022', |
| 131 | '--module', |
| 132 | 'NodeNext', |
| 133 | '--moduleResolution', |
| 134 | 'NodeNext', |
| 135 | 'consumer.ts', |
| 136 | ]); |
| 137 | yield* command('node', ['consumer.ts']); |
| 138 | }); |
| 139 | |
| 140 | /** Installs the candidate npm packages fresh and type-checks a strict consumer. */ |
| 141 | const verifyNpmConsumer = ( |
| 142 | cwd: string, |
| 143 | batch: ReleaseBatch, |
| 144 | directory: string, |
| 145 | candidates: string |
| 146 | ) => |
| 147 | Effect.gen(function* () { |
| 148 | const path = yield* Path.Path; |
| 149 | const command = inDirectory(directory); |
| 150 | const npm = batch.plan.releases.filter(release => release.language === 'typescript'); |
| 151 | const { packages } = yield* fromPromise(() => getPackages(cwd)); |
| 152 | const peers = yield* externalPeers(npm, packages); |
| 153 | const typescript = yield* installedVersion( |
| 154 | path.resolve(cwd, 'node_modules/typescript/package.json') |
| 155 | ); |
| 156 | const nodeTypes = yield* installedVersion( |
| 157 | path.resolve(cwd, 'node_modules/@types/node/package.json') |
| 158 | ); |
| 159 | yield* writeFile( |
| 160 | path.join(directory, 'package.json'), |
| 161 | JSON.stringify({ private: true, type: 'module' }) |
| 162 | ); |
| 163 | yield* command( |
| 164 | 'npm', |
| 165 | npmInstallArgs(candidates, npm, [ |
| 166 | `typescript@${typescript}`, |
| 167 | `@types/node@${nodeTypes}`, |
| 168 | ...[...peers].map(([name, version]) => `${name}@${version}`), |
| 169 | ]) |
| 170 | ); |
| 171 | for (const release of npm) { |
| 172 | const installed = yield* installedVersion( |
| 173 | path.join(directory, 'node_modules', release.name, 'package.json') |
| 174 | ); |
| 175 | yield* ensure( |
| 176 | installed === release.newVersion && |
| 177 | packages.some( |
| 178 | pkg => |
| 179 | pkg.packageJson.name === release.name && |
| 180 | pkg.packageJson.version === release.newVersion |
| 181 | ), |
| 182 | `Candidate consumer installed the wrong version: ${release.name}` |
| 183 | ); |
| 184 | } |
| 185 | const providers = (yield* fromPromise(() => discoverProviderCases())).filter(provider => |
| 186 | npm.some(release => release.name === provider.name) |
| 187 | ); |
| 188 | yield* checkStrictConsumer( |
| 189 | directory, |
| 190 | npm.map(release => release.name), |
| 191 | providers |
| 192 | ); |
| 193 | }); |
| 194 | |
| 195 | const pythonVerifier = (packages: ReadonlyArray<string>, version: string) => |
| 196 | `import importlib, importlib.metadata, pathlib\npackages = ${JSON.stringify(packages)}\nfor name in packages:\n distribution = importlib.metadata.distribution(name)\n assert distribution.version == ${JSON.stringify(version)}, (name, distribution.version)\n markers = [file for file in distribution.files or [] if str(file).endswith('/py.typed')]\n assert markers, ('Missing typing metadata', name)\n modules = {str(marker).split('/')[0] for marker in markers}\n for module in modules:\n importlib.import_module(module)\n assert pathlib.Path(distribution.locate_file(module), 'py.typed').exists()\nprint('Verified public imports, exact versions and typing metadata:', packages)\n`; |
| 197 | |
| 198 | const pythonVersion = (cwd: string) => |
| 199 | Effect.gen(function* () { |
| 200 | const fs = yield* FileSystem.FileSystem; |
| 201 | const path = yield* Path.Path; |
| 202 | const mise = yield* Effect.fromResult( |
| 203 | Toml.parse(yield* fs.readFileString(path.resolve(cwd, 'mise.toml'))) |
| 204 | ); |
| 205 | return (yield* Schema.decodeUnknownEffect(MiseTools)(mise)).tools.python; |
| 206 | }); |
| 207 | |
| 208 | const candidateWheels = (candidates: string) => |
| 209 | Effect.flatMap(FileSystem.FileSystem, fs => fs.readDirectory(candidates)); |
| 210 | |
| 211 | /** Each Python package gets its own environment: providers have incompatible dependency trains. */ |
| 212 | const verifyPythonConsumer = ( |
| 213 | cwd: string, |
| 214 | batch: ReleaseBatch, |
| 215 | directory: string, |
| 216 | candidates: string |
| 217 | ) => |
| 218 | Effect.gen(function* () { |
| 219 | const path = yield* Path.Path; |
| 220 | const command = inDirectory(directory); |
| 221 | const version = REGISTRY_VERSION.python( |
| 222 | batch.plan.releases.find(release => release.language === 'python')?.newVersion ?? '' |
| 223 | ); |
| 224 | const packages = yield* discoverPythonPackages(cwd); |
| 225 | const python = yield* pythonVersion(cwd); |
| 226 | const wheels = yield* candidateWheels(candidates); |
| 227 | const requirement = (name: string) => |
| 228 | Effect.fromNullishOr(pythonRequirement(candidates, wheels, name, version)).pipe( |
| 229 | Effect.mapError(() => new ReleaseError({ message: `Missing candidate wheel: ${name}` })) |
| 230 | ); |
| 231 | for (const pkg of packages) { |
| 232 | const environment = path.join(directory, pkg.name); |
| 233 | yield* command('uv', ['venv', '--python', python, environment]); |
| 234 | const executable = path.join(environment, 'bin/python'); |
| 235 | const names = [...new Set(['composio', pkg.name])]; |
| 236 | yield* command('uv', [ |
| 237 | 'pip', |
| 238 | 'install', |
| 239 | '--python', |
| 240 | executable, |
| 241 | '--index-url', |
| 242 | PYPI_INDEX, |
| 243 | ...(yield* Effect.forEach(names, requirement)), |
| 244 | ]); |
| 245 | yield* command('uv', ['pip', 'check', '--python', executable]); |
| 246 | yield* writeFile(path.join(environment, 'verify.py'), pythonVerifier(names, version)); |
| 247 | yield* command(executable, [path.join(environment, 'verify.py')]); |
| 248 | } |
| 249 | }); |
| 250 | |
| 251 | const providerImport = (name: string) => |
| 252 | `import importlib, importlib.metadata\ndistribution = importlib.metadata.distribution(${JSON.stringify(name)})\nmodules = {str(file).split('/')[0] for file in distribution.files or [] if str(file).endswith('/__init__.py')}\nassert modules, ('No importable modules', ${JSON.stringify(name)})\nfor module in sorted(modules):\n importlib.import_module(module)\nprint('Imported', ${JSON.stringify(name)}, distribution.version)\n`; |
| 253 | |
| 254 | /** |
| 255 | * Providers depend on an unpinned `composio`, so users can upgrade core alone: install the |
| 256 | * candidate core wheel with each provider's latest published version and import the provider. |
| 257 | */ |
| 258 | export const verifyPublishedProviders = ( |
| 259 | cwd: string, |
| 260 | candidates: string, |
| 261 | version: string, |
| 262 | directory: string |
| 263 | ) => |
| 264 | Effect.gen(function* () { |
| 265 | const path = yield* Path.Path; |
| 266 | const command = inDirectory(directory); |
| 267 | const python = yield* pythonVersion(cwd); |
| 268 | const core = pythonRequirement( |
| 269 | candidates, |
| 270 | yield* candidateWheels(candidates), |
| 271 | 'composio', |
| 272 | version |
| 273 | ); |
| 274 | if (!core) return yield* fail('Missing candidate wheel: composio'); |
| 275 | const providers = (yield* discoverPythonPackages(cwd)).filter(pkg => pkg.name !== 'composio'); |
| 276 | const failures: string[] = []; |
| 277 | const client = yield* HttpClient.HttpClient; |
| 278 | for (const provider of providers) { |
| 279 | // A provider released for the first time has no published version to check against yet. |
| 280 | const published = yield* client.get(`https://pypi.org/pypi/${provider.name}/json`); |
| 281 | if (published.status === 404) continue; |
| 282 | const environment = path.join(directory, `published-${provider.name}`); |
| 283 | const executable = path.join(environment, 'bin/python'); |
| 284 | yield* Effect.gen(function* () { |
| 285 | yield* command('uv', ['venv', '--python', python, environment]); |
| 286 | yield* command('uv', [ |
| 287 | 'pip', |
| 288 | 'install', |
| 289 | '--python', |
| 290 | executable, |
| 291 | '--index-url', |
| 292 | PYPI_INDEX, |
| 293 | core, |
| 294 | provider.name, |
| 295 | ]); |
| 296 | yield* writeFile(path.join(environment, 'verify.py'), providerImport(provider.name)); |
| 297 | yield* command(executable, [path.join(environment, 'verify.py')]); |
| 298 | }).pipe( |
| 299 | Effect.catch(error => |
| 300 | Effect.sync(() => |
| 301 | failures.push( |
| 302 | `Candidate composio breaks the latest published ${provider.name}: ${error.message}` |
| 303 | ) |
| 304 | ) |
| 305 | ) |
| 306 | ); |
| 307 | } |
| 308 | yield* ensure(failures.length === 0, failures.join('\n')); |
| 309 | }); |
| 310 | |
| 311 | const CONSUMERS: Record<Language, typeof verifyNpmConsumer | typeof verifyPythonConsumer> = { |
| 312 | typescript: verifyNpmConsumer, |
| 313 | python: verifyPythonConsumer, |
| 314 | }; |
| 315 | |
| 316 | export const readVerifiedCandidate = (candidates: string) => |
| 317 | Effect.gen(function* () { |
| 318 | const manifest = yield* readJson(CandidateManifest, `${candidates}/${CANDIDATE_MANIFEST}`); |
| 319 | const files = yield* readCandidateFiles(candidates); |
| 320 | yield* verifyCandidateFiles(files, manifest.files); |
| 321 | return { manifest, files }; |
| 322 | }); |
| 323 | |
| 324 | /** Checks a downloaded candidate against its manifest, then installs and imports it like a user. */ |
| 325 | export const verifyCandidateConsumers = (cwd: string, batch: ReleaseBatch, candidates: string) => |
| 326 | Effect.scoped( |
| 327 | Effect.gen(function* () { |
| 328 | const fs = yield* FileSystem.FileSystem; |
| 329 | const { manifest } = yield* readVerifiedCandidate(candidates); |
| 330 | const directory = yield* fs.makeTempDirectoryScoped({ prefix: 'sdk-candidate-consumer-' }); |
| 331 | yield* CONSUMERS[manifest.language](cwd, batch, directory, candidates); |
| 332 | const python = batch.plan.releases.find(release => release.language === 'python'); |
| 333 | if (manifest.language === 'python' && python) |
| 334 | yield* verifyPublishedProviders( |
| 335 | cwd, |
| 336 | candidates, |
| 337 | REGISTRY_VERSION.python(python.newVersion), |
| 338 | directory |
| 339 | ); |
| 340 | }) |
| 341 | ); |
| 342 | |
| 343 | /** `consumers.ts <candidate-dir>` installs a downloaded candidate like a user would. */ |
| 344 | const main = Effect.gen(function* () { |
| 345 | const path = yield* Path.Path; |
| 346 | const [candidates] = yield* Schema.decodeUnknownEffect(Schema.Tuple([Schema.String]))( |
| 347 | yield* (yield* Stdio.Stdio).args |
| 348 | ); |
| 349 | const batch = yield* readJson(ReleaseBatch, BATCH_PATH); |
| 350 | yield* prepareCandidateSource(path.resolve('.'), batch); |
| 351 | yield* verifyCandidateConsumers(path.resolve('.'), batch, path.resolve(candidates)); |
| 352 | }); |
| 353 | |
| 354 | if (import.meta.main) runMain(main); |