Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 04:45 UTC
| 1 | import { builtinModules } from 'node:module'; |
| 2 | import { mkdir, readdir, rm, stat, writeFile } from 'node:fs/promises'; |
| 3 | import * as path from 'node:path'; |
| 4 | import process from 'node:process'; |
| 5 | import { Console, Effect, Stream } from 'effect'; |
| 6 | import { ChildProcess as Command } from 'effect/process'; |
| 7 | import { RUN_COMPANION_MODULE_BASENAMES } from '../src/services/run-companion-modules'; |
| 8 | import { buildCliReleaseVersionDefineArgs } from '../src/utils/cli-release-version'; |
| 9 | |
| 10 | import { BinaryBuildError } from './build-error'; |
| 11 | import { compiledBinaryFileName, type ReleaseTarget } from './_release-artifacts'; |
| 12 | |
| 13 | export { teardown } from './_teardown'; |
| 14 | |
| 15 | const RUN_COMPANION_SERVICE_ENTRY_MAP = Object.fromEntries( |
| 16 | RUN_COMPANION_MODULE_BASENAMES.map(name => [`services/${name}`, `src/services/${name}.ts`]) |
| 17 | ) satisfies Record<string, string>; |
| 18 | |
| 19 | const allowedRuntimeSpecifiers = new Set( |
| 20 | builtinModules.flatMap(specifier => |
| 21 | specifier.startsWith('node:') |
| 22 | ? [specifier, specifier.slice('node:'.length)] |
| 23 | : [specifier, `node:${specifier}`] |
| 24 | ) |
| 25 | ); |
| 26 | |
| 27 | const importStatementPattern = /(?:^|[;\n])\s*import\s+(?:[^'"`\n]+?\s+from\s+)?["']([^"']+)["']/gm; |
| 28 | const exportStatementPattern = |
| 29 | /(?:^|[;\n])\s*export\s+(?:\*\s+from\s+|\{[^}\n]+\}\s+from\s+)["']([^"']+)["']/gm; |
| 30 | const runtimeImportCallPattern = /(?:^|[^\w$.])(?:__require|require|import)\(\s*["']([^"']+)["']/g; |
| 31 | |
| 32 | // Kept out of --env: Bun honors only the last --env, which would clobber DEBUG_OVERRIDE_*. |
| 33 | export const posthogBakeArgs = (): ReadonlyArray<string> => { |
| 34 | const key = process.env.COMPOSIO_POSTHOG_PROJECT_API_KEY?.trim(); |
| 35 | return key ? ['--define', `COMPOSIO_POSTHOG_PROJECT_API_KEY_BAKED=${JSON.stringify(key)}`] : []; |
| 36 | }; |
| 37 | |
| 38 | const isAllowedRuntimeSpecifier = (specifier: string): boolean => |
| 39 | specifier.startsWith('.') || |
| 40 | specifier.startsWith('/') || |
| 41 | specifier.startsWith('data:') || |
| 42 | specifier === 'bun' || |
| 43 | specifier.startsWith('bun:') || |
| 44 | allowedRuntimeSpecifiers.has(specifier); |
| 45 | |
| 46 | const stripStringsAndComments = (source: string): string => { |
| 47 | let result = ''; |
| 48 | let index = 0; |
| 49 | |
| 50 | const appendSpace = (char: string) => { |
| 51 | result += char === '\n' ? '\n' : ' '; |
| 52 | }; |
| 53 | |
| 54 | while (index < source.length) { |
| 55 | const char = source[index]; |
| 56 | const next = source[index + 1]; |
| 57 | |
| 58 | if (char === "'" || char === '"' || char === '`') { |
| 59 | const quote = char; |
| 60 | // The literal's body is dropped rather than blanked out. Blanking kept a |
| 61 | // same-length run of spaces, and once a bundle carries a multi-megabyte |
| 62 | // string with newlines in it (the TypeScript compiler's embedded lib |
| 63 | // files, in `generation-runtime`), the `^\s*` prefix of the import |
| 64 | // patterns below backtracks across that run from every line start — |
| 65 | // quadratic, and it stalled the build for over ten minutes. |
| 66 | result += ' '; |
| 67 | index += 1; |
| 68 | |
| 69 | while (index < source.length) { |
| 70 | const current = source[index]; |
| 71 | index += 1; |
| 72 | |
| 73 | if (current === '\\') { |
| 74 | index += 1; |
| 75 | continue; |
| 76 | } |
| 77 | |
| 78 | if (current === quote) { |
| 79 | break; |
| 80 | } |
| 81 | } |
| 82 | |
| 83 | result += ' '; |
| 84 | continue; |
| 85 | } |
| 86 | |
| 87 | if (char === '/' && next === '/') { |
| 88 | appendSpace(char); |
| 89 | appendSpace(next); |
| 90 | index += 2; |
| 91 | |
| 92 | while (index < source.length) { |
| 93 | const current = source[index]; |
| 94 | appendSpace(current); |
| 95 | index += 1; |
| 96 | if (current === '\n') { |
| 97 | break; |
| 98 | } |
| 99 | } |
| 100 | |
| 101 | continue; |
| 102 | } |
| 103 | |
| 104 | if (char === '/' && next === '*') { |
| 105 | appendSpace(char); |
| 106 | appendSpace(next); |
| 107 | index += 2; |
| 108 | |
| 109 | while (index < source.length) { |
| 110 | const current = source[index]; |
| 111 | const following = source[index + 1]; |
| 112 | appendSpace(current); |
| 113 | index += 1; |
| 114 | if (current === '*' && following === '/') { |
| 115 | appendSpace(following); |
| 116 | index += 1; |
| 117 | break; |
| 118 | } |
| 119 | } |
| 120 | |
| 121 | continue; |
| 122 | } |
| 123 | |
| 124 | result += char; |
| 125 | index += 1; |
| 126 | } |
| 127 | |
| 128 | return result; |
| 129 | }; |
| 130 | |
| 131 | const collectBundledSpecifiers = (source: string): ReadonlyArray<string> => { |
| 132 | const sanitizedSource = stripStringsAndComments(source); |
| 133 | const specifiers = new Set<string>(); |
| 134 | |
| 135 | for (const pattern of [ |
| 136 | importStatementPattern, |
| 137 | exportStatementPattern, |
| 138 | runtimeImportCallPattern, |
| 139 | ]) { |
| 140 | for (const match of sanitizedSource.matchAll(pattern)) { |
| 141 | const specifier = match[1]; |
| 142 | if (specifier) { |
| 143 | specifiers.add(specifier); |
| 144 | } |
| 145 | } |
| 146 | } |
| 147 | |
| 148 | return [...specifiers]; |
| 149 | }; |
| 150 | |
| 151 | const collectJavaScriptFiles = async (directory: string): Promise<ReadonlyArray<string>> => { |
| 152 | const collected: string[] = []; |
| 153 | |
| 154 | const walk = async (directory: string): Promise<void> => { |
| 155 | for (const entry of await readdir(directory, { withFileTypes: true })) { |
| 156 | const entryPath = path.join(directory, entry.name); |
| 157 | if (entry.isDirectory()) { |
| 158 | await walk(entryPath); |
| 159 | continue; |
| 160 | } |
| 161 | |
| 162 | if (/\.(?:m?js|cjs)$/.test(entry.name)) { |
| 163 | collected.push(entryPath); |
| 164 | } |
| 165 | } |
| 166 | }; |
| 167 | |
| 168 | await walk(directory); |
| 169 | return collected.sort(); |
| 170 | }; |
| 171 | |
| 172 | const collectRuntimeFiles = async (rootDir: string): Promise<ReadonlyArray<string>> => { |
| 173 | const collected = new Set<string>(); |
| 174 | |
| 175 | for (const baseName of RUN_COMPANION_MODULE_BASENAMES) { |
| 176 | const wrapperPath = path.join(rootDir, `${baseName}.mjs`); |
| 177 | if (await Bun.file(wrapperPath).exists()) { |
| 178 | collected.add(wrapperPath); |
| 179 | } |
| 180 | } |
| 181 | |
| 182 | const servicesDir = path.join(rootDir, 'services'); |
| 183 | const exists = await stat(servicesDir) |
| 184 | .then(stats => stats.isDirectory()) |
| 185 | .catch(() => false); |
| 186 | if (exists) { |
| 187 | for (const filePath of await collectJavaScriptFiles(servicesDir)) { |
| 188 | collected.add(filePath); |
| 189 | } |
| 190 | } |
| 191 | |
| 192 | return [...collected].sort(); |
| 193 | }; |
| 194 | |
| 195 | const assertBundledRuntimeFiles = async (rootDir: string): Promise<void> => { |
| 196 | const files = await collectRuntimeFiles(rootDir); |
| 197 | const violations: string[] = []; |
| 198 | |
| 199 | for (const filePath of files) { |
| 200 | const source = await Bun.file(filePath).text(); |
| 201 | for (const specifier of collectBundledSpecifiers(source)) { |
| 202 | if (!specifier || isAllowedRuntimeSpecifier(specifier)) { |
| 203 | continue; |
| 204 | } |
| 205 | |
| 206 | violations.push(`${path.relative(rootDir, filePath)} -> ${specifier}`); |
| 207 | } |
| 208 | } |
| 209 | |
| 210 | if (violations.length > 0) { |
| 211 | throw new Error( |
| 212 | [ |
| 213 | 'Generated runtime support files still reference external packages:', |
| 214 | ...violations.map(violation => ` - ${violation}`), |
| 215 | ].join('\n') |
| 216 | ); |
| 217 | } |
| 218 | }; |
| 219 | |
| 220 | const buildCompanionServiceBundles = async (outputDir: string): Promise<void> => { |
| 221 | const servicesOutputDir = path.join(outputDir, 'services'); |
| 222 | await rm(servicesOutputDir, { force: true, recursive: true }); |
| 223 | await mkdir(servicesOutputDir, { recursive: true }); |
| 224 | |
| 225 | for (const [entryName, entryPath] of Object.entries(RUN_COMPANION_SERVICE_ENTRY_MAP)) { |
| 226 | const relativeOutputPath = `${entryName}.mjs`; |
| 227 | const outputDirectory = path.join(outputDir, path.dirname(relativeOutputPath)); |
| 228 | const outputFileName = path.basename(relativeOutputPath); |
| 229 | await mkdir(outputDirectory, { recursive: true }); |
| 230 | |
| 231 | const result = await Bun.build({ |
| 232 | entrypoints: [path.resolve(entryPath)], |
| 233 | outdir: outputDirectory, |
| 234 | naming: outputFileName, |
| 235 | target: 'bun', |
| 236 | format: 'esm', |
| 237 | packages: 'bundle', |
| 238 | sourcemap: 'none', |
| 239 | }); |
| 240 | |
| 241 | if (!result.success) { |
| 242 | const details = result.logs |
| 243 | .map(log => log.message) |
| 244 | .filter(message => message.length > 0) |
| 245 | .join('\n'); |
| 246 | throw new Error( |
| 247 | details.length > 0 |
| 248 | ? `Failed to bundle ${entryName}:\n${details}` |
| 249 | : `Failed to bundle ${entryName}.` |
| 250 | ); |
| 251 | } |
| 252 | } |
| 253 | }; |
| 254 | |
| 255 | // Module paths that must only ever be reached through a companion module. The |
| 256 | // executable's bundle is checked after every build because the guard against |
| 257 | // regressions is otherwise invisible: a stray static import of any of these |
| 258 | // would silently put the TypeScript compiler back into the executable, and |
| 259 | // `--version` would quietly get ~70ms slower. |
| 260 | const EXECUTABLE_EXCLUDED_MODULE_PATTERNS: ReadonlyArray<{ |
| 261 | readonly pattern: RegExp; |
| 262 | readonly reason: string; |
| 263 | }> = [ |
| 264 | { pattern: /\/node_modules\/typescript\//, reason: 'the TypeScript compiler' }, |
| 265 | { pattern: /(?:^|\/)src\/generation\/(?!errors\.ts$)/, reason: 'the generation pipeline' }, |
| 266 | { |
| 267 | pattern: /(?:^|\/)src\/commands\/run-source-transforms\.ts$/, |
| 268 | reason: 'the run source rewrites', |
| 269 | }, |
| 270 | { pattern: /(?:^|\/)src\/services\/generation-runtime\.ts$/, reason: 'the generation companion' }, |
| 271 | ]; |
| 272 | |
| 273 | // `--define` pairs as `Bun.build` takes them. |
| 274 | const defineRecordFromArgs = (args: ReadonlyArray<string>): Record<string, string> => |
| 275 | Object.fromEntries( |
| 276 | args.flatMap((arg, index) => { |
| 277 | const pair = args[index - 1] === '--define' ? arg : undefined; |
| 278 | const separator = pair?.indexOf('=') ?? -1; |
| 279 | return pair && separator > 0 ? [[pair.slice(0, separator), pair.slice(separator + 1)]] : []; |
| 280 | }) |
| 281 | ); |
| 282 | |
| 283 | // Bun's unminified output starts each module with a `// <path>` line, which is |
| 284 | // the only bundle-level record of what made it into the graph. A dynamic |
| 285 | // `import()` of a *literal* specifier is still bundled (lazily evaluated, but |
| 286 | // parsed on every start), so this catches those too — only the runtime-computed |
| 287 | // specifier in `loadInstalledCompanionModule` keeps a module out. |
| 288 | const assertExecutableExcludesCompanionModules = async (): Promise<void> => { |
| 289 | const result = await Bun.build({ |
| 290 | entrypoints: [path.resolve('./src/bin.ts')], |
| 291 | target: 'bun', |
| 292 | format: 'esm', |
| 293 | packages: 'bundle', |
| 294 | sourcemap: 'none', |
| 295 | // The same environment inlining and defines as `build-binary.ts` and |
| 296 | // `build-binary-cross.ts`, plus the NODE_ENV that `--production` sets, so a |
| 297 | // branch the release build folds away is folded away here too. |
| 298 | env: 'DEBUG_OVERRIDE_*', |
| 299 | define: { |
| 300 | 'process.env.NODE_ENV': JSON.stringify('production'), |
| 301 | ...defineRecordFromArgs([ |
| 302 | ...posthogBakeArgs(), |
| 303 | ...buildCliReleaseVersionDefineArgs(process.env.RELEASE_TAG), |
| 304 | ]), |
| 305 | }, |
| 306 | // `--production` also minifies. Syntax minification is the part that drops |
| 307 | // unreachable code; whitespace minification would strip the module |
| 308 | // comments this check reads, so it stays off. |
| 309 | minify: { syntax: true, whitespace: false, identifiers: false }, |
| 310 | }); |
| 311 | if (!result.success) { |
| 312 | throw new Error( |
| 313 | `Failed to bundle src/bin.ts for the executable graph check:\n${result.logs |
| 314 | .map(log => log.message) |
| 315 | .join('\n')}` |
| 316 | ); |
| 317 | } |
| 318 | |
| 319 | const violations: string[] = []; |
| 320 | for (const artifact of result.outputs) { |
| 321 | const source = await artifact.text(); |
| 322 | for (const line of source.split('\n')) { |
| 323 | if (!line.startsWith('// ') || line.startsWith('// @bun')) { |
| 324 | continue; |
| 325 | } |
| 326 | const modulePath = line.slice(3); |
| 327 | const hit = EXECUTABLE_EXCLUDED_MODULE_PATTERNS.find(({ pattern }) => |
| 328 | pattern.test(modulePath) |
| 329 | ); |
| 330 | if (hit) { |
| 331 | violations.push(`${modulePath} (${hit.reason})`); |
| 332 | } |
| 333 | } |
| 334 | } |
| 335 | |
| 336 | if (violations.length > 0) { |
| 337 | throw new Error( |
| 338 | [ |
| 339 | 'The executable bundle reaches modules that must only be loaded through a companion module', |
| 340 | '(see RUN_COMPANION_MODULE_BASENAMES). Import them via `loadInstalledCompanionModule` instead:', |
| 341 | ...violations.map(violation => ` - ${violation}`), |
| 342 | ].join('\n') |
| 343 | ); |
| 344 | } |
| 345 | }; |
| 346 | |
| 347 | export const buildCompanionModules = (outputDir: string) => |
| 348 | Effect.gen(function* () { |
| 349 | yield* Effect.tryPromise(() => mkdir(outputDir, { recursive: true })); |
| 350 | |
| 351 | yield* Effect.tryPromise(() => buildCompanionServiceBundles(outputDir)); |
| 352 | |
| 353 | for (const name of RUN_COMPANION_MODULE_BASENAMES) { |
| 354 | const wrapperPath = path.join(outputDir, `${name}.mjs`); |
| 355 | const wrapperSource = `export * from "./services/${name}.mjs";\n`; |
| 356 | yield* Effect.tryPromise(() => writeFile(wrapperPath, wrapperSource, 'utf8')); |
| 357 | } |
| 358 | |
| 359 | yield* Effect.tryPromise(() => assertBundledRuntimeFiles(outputDir)); |
| 360 | yield* Effect.tryPromise(() => assertExecutableExcludesCompanionModules()); |
| 361 | }); |
| 362 | |
| 363 | /** |
| 364 | * Cross-compile the CLI for one release target. Returns the path of the compiled binary. |
| 365 | */ |
| 366 | export const compileBinary = ({ bunTarget, artifact }: ReleaseTarget) => |
| 367 | Effect.gen(function* () { |
| 368 | const outfile = `./dist/binaries/${compiledBinaryFileName(artifact)}`; |
| 369 | const args = [ |
| 370 | 'build', |
| 371 | './src/bin.ts', |
| 372 | '--env', |
| 373 | 'DEBUG_OVERRIDE_*', |
| 374 | ...posthogBakeArgs(), |
| 375 | ...buildCliReleaseVersionDefineArgs(process.env.RELEASE_TAG), |
| 376 | '--compile', |
| 377 | '--production', |
| 378 | '--target', |
| 379 | bunTarget, |
| 380 | '--outfile', |
| 381 | outfile, |
| 382 | ]; |
| 383 | yield* Effect.logDebug(`Running bun ${args.join(' ')}`); |
| 384 | |
| 385 | const child = yield* Command.make('bun', args); |
| 386 | const { exitCode } = yield* Effect.all( |
| 387 | { |
| 388 | exitCode: child.exitCode, |
| 389 | output: Stream.merge( |
| 390 | Stream.decodeText(child.stdout, { encoding: 'utf-8' }), |
| 391 | Stream.decodeText(child.stderr, { encoding: 'utf-8' }), |
| 392 | { haltStrategy: 'left' } |
| 393 | ).pipe( |
| 394 | Stream.tap(chunk => Console.log(chunk)), |
| 395 | Stream.runDrain |
| 396 | ), |
| 397 | }, |
| 398 | { concurrency: 'unbounded' } |
| 399 | ); |
| 400 | |
| 401 | if (exitCode !== 0) { |
| 402 | // `teardown` exits with this code. |
| 403 | process.exitCode = exitCode; |
| 404 | return yield* new BinaryBuildError({ |
| 405 | message: `Failed to build binary for ${bunTarget}`, |
| 406 | target: bunTarget, |
| 407 | exitCode, |
| 408 | }); |
| 409 | } |
| 410 | return outfile; |
| 411 | }); |