Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
test(cli): test validity of generated Python code
jkomyno committed
Jul 29, 2025 at 01:45 UTC
43cf59bbf5d411ec28b8d7009b8fc6aa339ed19e
5 files changed
+280
-3
ts/packages/cli/test/__utils__/python-compiler.ts
new
+255
@@ -0,0 +1,255 @@
1
+import { spawnSync } from 'child_process';
2
+import { describe, it, expect } from 'vitest';
3
+
4
+interface AssertPythonIsValidInput {
5
+ files: {
6
+ [filename: string]: string;
7
+ };
8
+}
9
+
10
+/**
11
+ * Asserts that the provided Python code is syntactically valid.
12
+ */
13
+export function assertPythonIsValid({ files }: AssertPythonIsValidInput) {
14
+ for (const [filename, code] of Object.entries(files)) {
15
+ try {
16
+ // Use spawnSync to avoid shell quote escaping issues by passing code via stdin
17
+ const {
18
+ status: exitCode,
19
+ stderr,
20
+ stdout,
21
+ } = spawnSync(
22
+ 'python3',
23
+ ['-c', 'import sys; compile(sys.stdin.read(), "<string>", "exec")'],
24
+ {
25
+ input: code,
26
+ encoding: 'utf8',
27
+ }
28
+ );
29
+
30
+ if (exitCode !== 0) {
31
+ throw new Error(stderr || stdout || 'Python compilation failed');
32
+ }
33
+ } catch (error) {
34
+ if (error instanceof Error) {
35
+ throw new Error(`Invalid Python code in ${filename}:\n${error.message}`);
36
+ }
37
+ throw error;
38
+ }
39
+ }
40
+}
41
+
42
+if (import.meta.vitest) {
43
+ describe('assertPythonIsValid', () => {
44
+ it('[Given] valid Python code [Then] no errors are found', () => {
45
+ const code = /* python */ `
46
+def greet(name):
47
+ return f"Hello, {name}!"
48
+
49
+result = greet("World")
50
+print(result)
51
+ `;
52
+ assertPythonIsValid({ files: { 'main.py': code } });
53
+ });
54
+
55
+ it('[Given] valid Python code with imports [Then] no errors are found', () => {
56
+ const mainSource = /* python */ `
57
+import math
58
+
59
+def calculate_area(radius):
60
+ return math.pi * radius ** 2
61
+
62
+area = calculate_area(5)
63
+print(f"Area: {area}")
64
+ `;
65
+
66
+ assertPythonIsValid({
67
+ files: { 'main.py': mainSource },
68
+ });
69
+ });
70
+
71
+ it('[Given] valid Python code with class definition [Then] no errors are found', () => {
72
+ const code = /* python */ `
73
+class Calculator:
74
+ def __init__(self):
75
+ self.result = 0
76
+
77
+ def add(self, x, y):
78
+ self.result = x + y
79
+ return self.result
80
+
81
+calc = Calculator()
82
+calc.add(2, 3)
83
+ `;
84
+ assertPythonIsValid({ files: { 'calculator.py': code } });
85
+ });
86
+
87
+ it('[Given] valid Python code with string escaping [Then] no errors are found', () => {
88
+ const code = /* python */ `
89
+message = "Hello \\"World\\" with 'quotes'"
90
+template = '''This is a multiline
91
+string with "double" and 'single' quotes'''
92
+formatted = f"Template: {template}"
93
+print(formatted)
94
+ `;
95
+ assertPythonIsValid({ files: { 'strings.py': code } });
96
+ });
97
+
98
+ it('[Given] invalid Python code [Then] errors are found', () => {
99
+ const code = /* python */ `
100
+def broken_function(
101
+ print("This is missing a closing parenthesis"
102
+ `;
103
+ expect(() => {
104
+ assertPythonIsValid({ files: { 'broken.py': code } });
105
+ }).toThrowError();
106
+ });
107
+
108
+ it('[Given] Python code with syntax error [Then] errors are found', () => {
109
+ const code = /* python */ `
110
+def invalid_syntax():
111
+ if True
112
+ print("Missing colon")
113
+ `;
114
+ expect(() => {
115
+ assertPythonIsValid({ files: { 'syntax_error.py': code } });
116
+ }).toThrowError();
117
+ });
118
+
119
+ it('[Given] Python code with indentation error [Then] errors are found', () => {
120
+ const code = /* python */ `
121
+def bad_indentation():
122
+ x = 1
123
+ y = 2 # Wrong indentation
124
+ return x + y
125
+ `;
126
+ expect(() => {
127
+ assertPythonIsValid({ files: { 'indentation_error.py': code } });
128
+ }).toThrowError();
129
+ });
130
+
131
+ it('[Given] multiple valid Python files [Then] no errors are found', () => {
132
+ const mainSource = /* python */ `
133
+from math_utils import add, multiply
134
+
135
+def main():
136
+ result = multiply(add(2, 3), 4)
137
+ print(f"Result: {result}")
138
+
139
+if __name__ == "__main__":
140
+ main()
141
+ `;
142
+
143
+ const mathUtilsSource = /* python */ `
144
+def add(x, y):
145
+ return x + y
146
+
147
+def multiply(x, y):
148
+ return x * y
149
+ `;
150
+
151
+ assertPythonIsValid({
152
+ files: { 'main.py': mainSource, 'math_utils.py': mathUtilsSource },
153
+ });
154
+ });
155
+
156
+ it('[Given] valid Python code with decorators [Then] no errors are found', () => {
157
+ const code = /* python */ `
158
+def my_decorator(func):
159
+ def wrapper(*args, **kwargs):
160
+ print(f"Calling {func.__name__}")
161
+ return func(*args, **kwargs)
162
+ return wrapper
163
+
164
+@my_decorator
165
+def greet(name):
166
+ return f"Hello, {name}!"
167
+
168
+result = greet("World")
169
+print(result)
170
+ `;
171
+ assertPythonIsValid({ files: { 'decorators.py': code } });
172
+ });
173
+
174
+ it('[Given] valid Python code with async/await [Then] no errors are found', () => {
175
+ const code = /* python */ `
176
+import asyncio
177
+
178
+async def fetch_data():
179
+ await asyncio.sleep(1)
180
+ return "data"
181
+
182
+async def main():
183
+ result = await fetch_data()
184
+ print(result)
185
+
186
+if __name__ == "__main__":
187
+ asyncio.run(main())
188
+ `;
189
+ assertPythonIsValid({ files: { 'async_example.py': code } });
190
+ });
191
+
192
+ it('[Given] valid Python code with type hints [Then] no errors are found', () => {
193
+ const code = /* python */ `
194
+from typing import List, Optional, Dict
195
+
196
+def process_items(items: List[str], config: Optional[Dict[str, bool]] = None) -> str:
197
+ if config is None:
198
+ config = {}
199
+
200
+ processed = [item.upper() for item in items if config.get("uppercase", True)]
201
+ return ", ".join(processed)
202
+
203
+result: str = process_items(["hello", "world"])
204
+print(result)
205
+ `;
206
+ assertPythonIsValid({ files: { 'type_hints.py': code } });
207
+ });
208
+
209
+ it('[Given] Python code with complex string literals [Then] no errors are found', () => {
210
+ const code = /* python */ `
211
+# Test various string literal formats
212
+single_quotes = 'This is a single-quoted string'
213
+double_quotes = "This is a double-quoted string"
214
+triple_single = '''This is a
215
+multi-line string
216
+with single quotes'''
217
+triple_double = """This is a
218
+multi-line string
219
+with double quotes"""
220
+
221
+# Raw strings
222
+raw_string = r"This is a raw string with \\n and \\t"
223
+f_string = f"Hello, {single_quotes}"
224
+
225
+# Mixed quotes
226
+mixed = "She said, 'Hello there!' to him"
227
+escaped = "This has \\"escaped\\" quotes"
228
+
229
+print(f_string)
230
+ `;
231
+ assertPythonIsValid({ files: { 'string_literals.py': code } });
232
+ });
233
+
234
+ it('[Given] invalid Python code with undefined variable [Then] errors are found', () => {
235
+ const code = /* python */ `
236
+def test_function():
237
+ return undefined_variable
238
+ `;
239
+ // Note: Python syntax validation won't catch undefined variables - only syntax errors
240
+ // This should pass syntax validation but would fail at runtime
241
+ assertPythonIsValid({ files: { 'undefined_var.py': code } });
242
+ });
243
+
244
+ it('[Given] Python code with syntax error in function call [Then] errors are found', () => {
245
+ const code = /* python */ `
246
+def test_function():
247
+ print("Hello",) # This trailing comma is valid
248
+ print("Hello",,) # This double comma is invalid
249
+ `;
250
+ expect(() => {
251
+ assertPythonIsValid({ files: { 'function_call_error.py': code } });
252
+ }).toThrowError();
253
+ });
254
+ });
255
+}
ts/packages/cli/test/__utils__/typescript-compiler.ts
-2
@@ -1,9 +1,7 @@
1
-import path from 'node:path';
1
import { describe, it, expect } from 'vitest';
2
import { Effect, Stream, String } from 'effect';
3
import { Command } from '@effect/platform';
4
import ts from 'typescript';
6
-import { cwd } from 'node:process';
5
6
interface AssertTypeScriptIsValidInput {
7
files: {
ts/packages/cli/test/src/commands/py/py.generate.cmd.test.ts
+19
@@ -7,6 +7,7 @@ import { makeTestToolkits } from 'test/__utils__/models/toolkits';
7
import { NodeProcess } from 'src/services/node-process';
8
import { TRIGGER_TYPE_GMAIL } from 'test/__mocks__/trigger-type-gmail';
9
import { TestLiveInput } from 'test/__utils__/services/test-layer';
10
+import { assertPythonIsValid } from 'test/__utils__/python-compiler';
11
12
describe('CLI: composio py generate', () => {
13
const appClientData = {
@@ -89,6 +90,15 @@ describe('CLI: composio py generate', () => {
90
pass
91
"
92
`);
93
+
94
+ assertPythonIsValid({ files: { 'gmail.py': gmailSourceCode } });
95
+ assertPythonIsValid({ files: { 'slack.py': slackSourceCode } });
96
+ assertPythonIsValid({
97
+ files: {
98
+ 'gmail.py': gmailSourceCode,
99
+ 'slack.py': slackSourceCode,
100
+ },
101
+ });
102
})
103
);
104
@@ -275,6 +285,15 @@ describe('CLI: composio py generate', () => {
285
pass
286
"
287
`);
288
+
289
+ assertPythonIsValid({ files: { 'gmail.py': gmailSourceCode } });
290
+ assertPythonIsValid({ files: { 'slack.py': slackSourceCode } });
291
+ assertPythonIsValid({
292
+ files: {
293
+ 'gmail.py': gmailSourceCode,
294
+ 'slack.py': slackSourceCode,
295
+ },
296
+ });
297
})
298
);
299
});
ts/packages/cli/test/src/generation/python/generate-toolkit-source.test.ts
+5
@@ -3,6 +3,7 @@ import { generatePythonToolkitSources } from 'src/generation/python/generate-too
3
import { createToolkitIndex } from 'src/generation/create-toolkit-index';
4
import { BANNER } from 'src/generation/constants';
5
import { makeTestToolkits } from 'test/__utils__/models/toolkits';
6
+import { assertPythonIsValid } from 'test/__utils__/python-compiler';
7
import { TRIGGER_TYPE_GMAIL } from 'test/__mocks__/trigger-type-gmail';
8
9
describe('generatePythonToolkitSources', () => {
@@ -51,6 +52,8 @@ describe('generatePythonToolkitSources', () => {
52
pass
53
"
54
`);
55
+
56
+ assertPythonIsValid({ files: Object.fromEntries(sources) });
57
});
58
59
it('[Given] toolkits with tools and triggerTypes [Then] it returns a Python source file for each toolkit', () => {
@@ -247,5 +250,7 @@ describe('generatePythonToolkitSources', () => {
250
pass
251
"
252
`);
253
+
254
+ assertPythonIsValid({ files: Object.fromEntries(sources) });
255
});
256
});
ts/packages/cli/vitest.config.ts
+1
-1
@@ -17,7 +17,7 @@ export default defineConfig({
17
},
18
include: ['test/**/*.test.ts'],
19
// When defined, Vitest will run all matched files with import.meta.vitest inside.
20
- includeSource: ['src/**/*.ts'],
20
+ includeSource: ['src/**/*.ts', 'test/__utils__/*-compiler.ts'],
21
unstubEnvs: true,
22
globalSetup: './test/__utils__/vitest.global-setup.ts',
23
},