@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 04:45 UTC

next
ts 411 lines 13.2 KB
Raw
1 import { builtinModules } from 'node:module';
2 import { mkdir, readdir, rm, stat, writeFile } from 'node:fs/promises';
3 import * as path from 'node:path';
4 import process from 'node:process';
5 import { Console, Effect, Stream } from 'effect';
6 import { ChildProcess as Command } from 'effect/process';
7 import { RUN_COMPANION_MODULE_BASENAMES } from '../src/services/run-companion-modules';
8 import { buildCliReleaseVersionDefineArgs } from '../src/utils/cli-release-version';
9
10 import { BinaryBuildError } from './build-error';
11 import { compiledBinaryFileName, type ReleaseTarget } from './_release-artifacts';
12
13 export { teardown } from './_teardown';
14
15 const RUN_COMPANION_SERVICE_ENTRY_MAP = Object.fromEntries(
16 RUN_COMPANION_MODULE_BASENAMES.map(name => [`services/${name}`, `src/services/${name}.ts`])
17 ) satisfies Record<string, string>;
18
19 const allowedRuntimeSpecifiers = new Set(
20 builtinModules.flatMap(specifier =>
21 specifier.startsWith('node:')
22 ? [specifier, specifier.slice('node:'.length)]
23 : [specifier, `node:${specifier}`]
24 )
25 );
26
27 const importStatementPattern = /(?:^|[;\n])\s*import\s+(?:[^'"`\n]+?\s+from\s+)?["']([^"']+)["']/gm;
28 const exportStatementPattern =
29 /(?:^|[;\n])\s*export\s+(?:\*\s+from\s+|\{[^}\n]+\}\s+from\s+)["']([^"']+)["']/gm;
30 const runtimeImportCallPattern = /(?:^|[^\w$.])(?:__require|require|import)\(\s*["']([^"']+)["']/g;
31
32 // Kept out of --env: Bun honors only the last --env, which would clobber DEBUG_OVERRIDE_*.
33 export const posthogBakeArgs = (): ReadonlyArray<string> => {
34 const key = process.env.COMPOSIO_POSTHOG_PROJECT_API_KEY?.trim();
35 return key ? ['--define', `COMPOSIO_POSTHOG_PROJECT_API_KEY_BAKED=${JSON.stringify(key)}`] : [];
36 };
37
38 const isAllowedRuntimeSpecifier = (specifier: string): boolean =>
39 specifier.startsWith('.') ||
40 specifier.startsWith('/') ||
41 specifier.startsWith('data:') ||
42 specifier === 'bun' ||
43 specifier.startsWith('bun:') ||
44 allowedRuntimeSpecifiers.has(specifier);
45
46 const stripStringsAndComments = (source: string): string => {
47 let result = '';
48 let index = 0;
49
50 const appendSpace = (char: string) => {
51 result += char === '\n' ? '\n' : ' ';
52 };
53
54 while (index < source.length) {
55 const char = source[index];
56 const next = source[index + 1];
57
58 if (char === "'" || char === '"' || char === '`') {
59 const quote = char;
60 // The literal's body is dropped rather than blanked out. Blanking kept a
61 // same-length run of spaces, and once a bundle carries a multi-megabyte
62 // string with newlines in it (the TypeScript compiler's embedded lib
63 // files, in `generation-runtime`), the `^\s*` prefix of the import
64 // patterns below backtracks across that run from every line start —
65 // quadratic, and it stalled the build for over ten minutes.
66 result += ' ';
67 index += 1;
68
69 while (index < source.length) {
70 const current = source[index];
71 index += 1;
72
73 if (current === '\\') {
74 index += 1;
75 continue;
76 }
77
78 if (current === quote) {
79 break;
80 }
81 }
82
83 result += ' ';
84 continue;
85 }
86
87 if (char === '/' && next === '/') {
88 appendSpace(char);
89 appendSpace(next);
90 index += 2;
91
92 while (index < source.length) {
93 const current = source[index];
94 appendSpace(current);
95 index += 1;
96 if (current === '\n') {
97 break;
98 }
99 }
100
101 continue;
102 }
103
104 if (char === '/' && next === '*') {
105 appendSpace(char);
106 appendSpace(next);
107 index += 2;
108
109 while (index < source.length) {
110 const current = source[index];
111 const following = source[index + 1];
112 appendSpace(current);
113 index += 1;
114 if (current === '*' && following === '/') {
115 appendSpace(following);
116 index += 1;
117 break;
118 }
119 }
120
121 continue;
122 }
123
124 result += char;
125 index += 1;
126 }
127
128 return result;
129 };
130
131 const collectBundledSpecifiers = (source: string): ReadonlyArray<string> => {
132 const sanitizedSource = stripStringsAndComments(source);
133 const specifiers = new Set<string>();
134
135 for (const pattern of [
136 importStatementPattern,
137 exportStatementPattern,
138 runtimeImportCallPattern,
139 ]) {
140 for (const match of sanitizedSource.matchAll(pattern)) {
141 const specifier = match[1];
142 if (specifier) {
143 specifiers.add(specifier);
144 }
145 }
146 }
147
148 return [...specifiers];
149 };
150
151 const collectJavaScriptFiles = async (directory: string): Promise<ReadonlyArray<string>> => {
152 const collected: string[] = [];
153
154 const walk = async (directory: string): Promise<void> => {
155 for (const entry of await readdir(directory, { withFileTypes: true })) {
156 const entryPath = path.join(directory, entry.name);
157 if (entry.isDirectory()) {
158 await walk(entryPath);
159 continue;
160 }
161
162 if (/\.(?:m?js|cjs)$/.test(entry.name)) {
163 collected.push(entryPath);
164 }
165 }
166 };
167
168 await walk(directory);
169 return collected.sort();
170 };
171
172 const collectRuntimeFiles = async (rootDir: string): Promise<ReadonlyArray<string>> => {
173 const collected = new Set<string>();
174
175 for (const baseName of RUN_COMPANION_MODULE_BASENAMES) {
176 const wrapperPath = path.join(rootDir, `${baseName}.mjs`);
177 if (await Bun.file(wrapperPath).exists()) {
178 collected.add(wrapperPath);
179 }
180 }
181
182 const servicesDir = path.join(rootDir, 'services');
183 const exists = await stat(servicesDir)
184 .then(stats => stats.isDirectory())
185 .catch(() => false);
186 if (exists) {
187 for (const filePath of await collectJavaScriptFiles(servicesDir)) {
188 collected.add(filePath);
189 }
190 }
191
192 return [...collected].sort();
193 };
194
195 const assertBundledRuntimeFiles = async (rootDir: string): Promise<void> => {
196 const files = await collectRuntimeFiles(rootDir);
197 const violations: string[] = [];
198
199 for (const filePath of files) {
200 const source = await Bun.file(filePath).text();
201 for (const specifier of collectBundledSpecifiers(source)) {
202 if (!specifier || isAllowedRuntimeSpecifier(specifier)) {
203 continue;
204 }
205
206 violations.push(`${path.relative(rootDir, filePath)} -> ${specifier}`);
207 }
208 }
209
210 if (violations.length > 0) {
211 throw new Error(
212 [
213 'Generated runtime support files still reference external packages:',
214 ...violations.map(violation => ` - ${violation}`),
215 ].join('\n')
216 );
217 }
218 };
219
220 const buildCompanionServiceBundles = async (outputDir: string): Promise<void> => {
221 const servicesOutputDir = path.join(outputDir, 'services');
222 await rm(servicesOutputDir, { force: true, recursive: true });
223 await mkdir(servicesOutputDir, { recursive: true });
224
225 for (const [entryName, entryPath] of Object.entries(RUN_COMPANION_SERVICE_ENTRY_MAP)) {
226 const relativeOutputPath = `${entryName}.mjs`;
227 const outputDirectory = path.join(outputDir, path.dirname(relativeOutputPath));
228 const outputFileName = path.basename(relativeOutputPath);
229 await mkdir(outputDirectory, { recursive: true });
230
231 const result = await Bun.build({
232 entrypoints: [path.resolve(entryPath)],
233 outdir: outputDirectory,
234 naming: outputFileName,
235 target: 'bun',
236 format: 'esm',
237 packages: 'bundle',
238 sourcemap: 'none',
239 });
240
241 if (!result.success) {
242 const details = result.logs
243 .map(log => log.message)
244 .filter(message => message.length > 0)
245 .join('\n');
246 throw new Error(
247 details.length > 0
248 ? `Failed to bundle ${entryName}:\n${details}`
249 : `Failed to bundle ${entryName}.`
250 );
251 }
252 }
253 };
254
255 // Module paths that must only ever be reached through a companion module. The
256 // executable's bundle is checked after every build because the guard against
257 // regressions is otherwise invisible: a stray static import of any of these
258 // would silently put the TypeScript compiler back into the executable, and
259 // `--version` would quietly get ~70ms slower.
260 const EXECUTABLE_EXCLUDED_MODULE_PATTERNS: ReadonlyArray<{
261 readonly pattern: RegExp;
262 readonly reason: string;
263 }> = [
264 { pattern: /\/node_modules\/typescript\//, reason: 'the TypeScript compiler' },
265 { pattern: /(?:^|\/)src\/generation\/(?!errors\.ts$)/, reason: 'the generation pipeline' },
266 {
267 pattern: /(?:^|\/)src\/commands\/run-source-transforms\.ts$/,
268 reason: 'the run source rewrites',
269 },
270 { pattern: /(?:^|\/)src\/services\/generation-runtime\.ts$/, reason: 'the generation companion' },
271 ];
272
273 // `--define` pairs as `Bun.build` takes them.
274 const defineRecordFromArgs = (args: ReadonlyArray<string>): Record<string, string> =>
275 Object.fromEntries(
276 args.flatMap((arg, index) => {
277 const pair = args[index - 1] === '--define' ? arg : undefined;
278 const separator = pair?.indexOf('=') ?? -1;
279 return pair && separator > 0 ? [[pair.slice(0, separator), pair.slice(separator + 1)]] : [];
280 })
281 );
282
283 // Bun's unminified output starts each module with a `// <path>` line, which is
284 // the only bundle-level record of what made it into the graph. A dynamic
285 // `import()` of a *literal* specifier is still bundled (lazily evaluated, but
286 // parsed on every start), so this catches those too — only the runtime-computed
287 // specifier in `loadInstalledCompanionModule` keeps a module out.
288 const assertExecutableExcludesCompanionModules = async (): Promise<void> => {
289 const result = await Bun.build({
290 entrypoints: [path.resolve('./src/bin.ts')],
291 target: 'bun',
292 format: 'esm',
293 packages: 'bundle',
294 sourcemap: 'none',
295 // The same environment inlining and defines as `build-binary.ts` and
296 // `build-binary-cross.ts`, plus the NODE_ENV that `--production` sets, so a
297 // branch the release build folds away is folded away here too.
298 env: 'DEBUG_OVERRIDE_*',
299 define: {
300 'process.env.NODE_ENV': JSON.stringify('production'),
301 ...defineRecordFromArgs([
302 ...posthogBakeArgs(),
303 ...buildCliReleaseVersionDefineArgs(process.env.RELEASE_TAG),
304 ]),
305 },
306 // `--production` also minifies. Syntax minification is the part that drops
307 // unreachable code; whitespace minification would strip the module
308 // comments this check reads, so it stays off.
309 minify: { syntax: true, whitespace: false, identifiers: false },
310 });
311 if (!result.success) {
312 throw new Error(
313 `Failed to bundle src/bin.ts for the executable graph check:\n${result.logs
314 .map(log => log.message)
315 .join('\n')}`
316 );
317 }
318
319 const violations: string[] = [];
320 for (const artifact of result.outputs) {
321 const source = await artifact.text();
322 for (const line of source.split('\n')) {
323 if (!line.startsWith('// ') || line.startsWith('// @bun')) {
324 continue;
325 }
326 const modulePath = line.slice(3);
327 const hit = EXECUTABLE_EXCLUDED_MODULE_PATTERNS.find(({ pattern }) =>
328 pattern.test(modulePath)
329 );
330 if (hit) {
331 violations.push(`${modulePath} (${hit.reason})`);
332 }
333 }
334 }
335
336 if (violations.length > 0) {
337 throw new Error(
338 [
339 'The executable bundle reaches modules that must only be loaded through a companion module',
340 '(see RUN_COMPANION_MODULE_BASENAMES). Import them via `loadInstalledCompanionModule` instead:',
341 ...violations.map(violation => ` - ${violation}`),
342 ].join('\n')
343 );
344 }
345 };
346
347 export const buildCompanionModules = (outputDir: string) =>
348 Effect.gen(function* () {
349 yield* Effect.tryPromise(() => mkdir(outputDir, { recursive: true }));
350
351 yield* Effect.tryPromise(() => buildCompanionServiceBundles(outputDir));
352
353 for (const name of RUN_COMPANION_MODULE_BASENAMES) {
354 const wrapperPath = path.join(outputDir, `${name}.mjs`);
355 const wrapperSource = `export * from "./services/${name}.mjs";\n`;
356 yield* Effect.tryPromise(() => writeFile(wrapperPath, wrapperSource, 'utf8'));
357 }
358
359 yield* Effect.tryPromise(() => assertBundledRuntimeFiles(outputDir));
360 yield* Effect.tryPromise(() => assertExecutableExcludesCompanionModules());
361 });
362
363 /**
364 * Cross-compile the CLI for one release target. Returns the path of the compiled binary.
365 */
366 export const compileBinary = ({ bunTarget, artifact }: ReleaseTarget) =>
367 Effect.gen(function* () {
368 const outfile = `./dist/binaries/${compiledBinaryFileName(artifact)}`;
369 const args = [
370 'build',
371 './src/bin.ts',
372 '--env',
373 'DEBUG_OVERRIDE_*',
374 ...posthogBakeArgs(),
375 ...buildCliReleaseVersionDefineArgs(process.env.RELEASE_TAG),
376 '--compile',
377 '--production',
378 '--target',
379 bunTarget,
380 '--outfile',
381 outfile,
382 ];
383 yield* Effect.logDebug(`Running bun ${args.join(' ')}`);
384
385 const child = yield* Command.make('bun', args);
386 const { exitCode } = yield* Effect.all(
387 {
388 exitCode: child.exitCode,
389 output: Stream.merge(
390 Stream.decodeText(child.stdout, { encoding: 'utf-8' }),
391 Stream.decodeText(child.stderr, { encoding: 'utf-8' }),
392 { haltStrategy: 'left' }
393 ).pipe(
394 Stream.tap(chunk => Console.log(chunk)),
395 Stream.runDrain
396 ),
397 },
398 { concurrency: 'unbounded' }
399 );
400
401 if (exitCode !== 0) {
402 // `teardown` exits with this code.
403 process.exitCode = exitCode;
404 return yield* new BinaryBuildError({
405 message: `Failed to build binary for ${bunTarget}`,
406 target: bunTarget,
407 exitCode,
408 });
409 }
410 return outfile;
411 });