@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 186 lines 8.41 KB
Raw
1 /**
2 * Python-enabled regression gate. Runs the real helper, local SDK/providers, and
3 * locked mypy; deliberately separate from Bun's Python-free static test suite.
4 * Run from docs/: bun scripts/python-snippets/test-helper.ts
5 */
6 import assert from 'node:assert/strict';
7 import { runPythonHelper, pythonSnippetViolations, type PythonSnippet } from '../check-python-snippets';
8
9 type Fixture = {
10 id: string;
11 code: string;
12 /** Expected snippet-relative SDK error line and mypy code, not recorded prose. */
13 api?: [number, string][];
14 syntax?: number[];
15 /** Expected rejected `# type: ignore` lines, with mypy's code when it reports one. */
16 directives?: [number, string | undefined][];
17 typeCheck?: boolean;
18 other?: RegExp;
19 };
20
21 const fixtures: Fixture[] = [
22 { id: 'control', code: 'composio.tools.exectue("X", user_id="u", arguments={})', api: [[1, 'attr-defined']] },
23 { id: 'constructor', code: 'Composio(apikey="bad")', api: [[1, 'call-overload']] },
24 {
25 id: 'constructor_import', code: 'from composio import Composio\nComposio(apikey="bad")',
26 api: [[2, 'call-overload']],
27 },
28 {
29 id: 'constructor_alias', code: 'from composio import Composio as C\nC(apikey="bad")',
30 api: [[2, 'call-overload']],
31 },
32 { id: 'instance_alias', code: 'c = Composio()\nc.create(user="u")', api: [[2, 'call-overload']] },
33 {
34 id: 'import_alias', code: 'from composio import Composio as C\nc = C()\nc.create(user="u")',
35 api: [[3, 'call-overload']],
36 },
37 {
38 id: 'typed_parameter', code: 'def example(c: Composio) -> None:\n c.create(user="u")',
39 api: [[2, 'call-overload']],
40 },
41 {
42 id: 'class_collision',
43 code: 'class Tools:\n pass\ncomposio.tools.exectue("X", user_id="u", arguments={})\nTools().not_real()',
44 api: [[3, 'attr-defined']], other: /"Tools" has no attribute "not_real"/,
45 },
46 {
47 id: 'nested_parameter',
48 code: 'def unrelated(composio: int) -> None:\n composio.not_real()\ncomposio.tools.exectue("X", user_id="u", arguments={})',
49 api: [[3, 'attr-defined']], other: /"int" has no attribute "not_real"/,
50 },
51 {
52 id: 'package_named_parameters',
53 code: 'def unrelated(composio_client: int, composio_openai: int) -> None:\n composio_client.not_real()\n composio_openai.not_real()',
54 other: /"int" has no attribute "not_real"/,
55 },
56 {
57 id: 'nested_local',
58 code: 'def unrelated() -> None:\n composio = 1\ncomposio.tools.exectue("X", user_id="u", arguments={})',
59 api: [[3, 'attr-defined']],
60 },
61 {
62 id: 'nested_constructor_name',
63 code: 'def unrelated(Composio: int) -> None:\n pass\nComposio(apikey="bad")', api: [[3, 'call-overload']],
64 },
65 {
66 id: 'comprehension_binding',
67 code: 'values = [composio for composio in (1,)]\ncomposio.tools.exectue("X", user_id="u", arguments={})',
68 api: [[2, 'attr-defined']],
69 },
70 {
71 id: 'invalid_method', code: 'method = "NOT_AN_HTTP_METHOD"\ncomposio.tools.proxy(endpoint="/me", method=method)',
72 api: [[2, 'arg-type']],
73 },
74 {
75 id: 'invalid_dictionary', code: 'composio.auth_configs.create(toolkit="gmail", options={"totally_wrong": 1})',
76 api: [[1, 'call-overload']],
77 },
78 {
79 id: 'invalid_literal_dictionary',
80 code: 'from composio.types import auth_scheme\nauth_scheme.api_key({"totally_wrong": 1})',
81 api: [[2, 'arg-type']],
82 },
83 {
84 id: 'provider', code: 'from composio_openai import OpenAIProvider\np = OpenAIProvider()\np.not_real()',
85 api: [[3, 'attr-defined']],
86 },
87 {
88 id: 'sdk_missing_import', code: 'from composio.missing import Missing', api: [[1, 'import-not-found']],
89 },
90 {
91 id: 'sdk_missing_export', code: 'from composio import Missing', api: [[1, 'attr-defined']],
92 },
93 {
94 id: 'sdk_module_alias', code: 'import composio as sdk\nsdk.not_real()', api: [[2, 'attr-defined']],
95 },
96 {
97 id: 'non_sdk_attribute_in_sdk_chain',
98 code: 'from composio.types import Tool\ndef transform(schema: Tool) -> None:\n schema.input_parameters["properties"].pop("page", None)',
99 other: /"object" has no attribute "pop"/,
100 },
101 {
102 id: 'upload_context_callback',
103 code: 'from composio import before_file_upload\n@before_file_upload(tools=["GOOGLEDRIVE_UPLOAD_FILE"])\ndef audit_path(context) -> str | bool:\n return context["path"]',
104 },
105 {
106 id: 'future_import',
107 code: 'from __future__ import annotations\ncomposio.tools.exectue("X", user_id="u", arguments={})',
108 api: [[2, 'attr-defined']],
109 },
110 { id: 'top_level_shadow', code: 'composio = "local"\ncomposio.not_real()', other: /"str" has no attribute/ },
111 {
112 id: 'third_party_alias', code: 'from pathlib import Path as Composio\nComposio(1)',
113 other: /\[arg-type\]/,
114 },
115 { id: 'unknown_provider', code: 'import composio_custom_provider', other: /\[import-not-found\]/ },
116 {
117 id: 'local_class', code: 'class Composio:\n pass\nComposio(apikey="bad")',
118 other: /\[call-arg\]/,
119 },
120 { id: 'legacy', code: 'from composio import ComposioToolSet\nComposioToolSet(apikey="x")', typeCheck: false },
121 { id: 'legacy_syntax', code: 'if True print("bad")', typeCheck: false, syntax: [1] },
122 { id: 'valid', code: 'c = Composio(api_key="test")\nc.create(user_id="u")' },
123 { id: 'top_level_await', code: 'await unknown_async_function()', other: /\[name-defined\]/ },
124 {
125 id: 'sdk_model_index',
126 code: 'r = composio.tools.proxy(endpoint="/me", method="GET", connected_account_id="ca_1")\nprint(r["status"])',
127 api: [[2, 'index']],
128 },
129 { id: 'local_index', code: 'value = 1\nprint(value[0])', other: /\[index\]/ },
130 {
131 id: 'sdk_typeddict_union_key',
132 code: 'parsed = composio.triggers.parse(body=b"", headers={}, verify_secret="s")\nprint(parsed["raw_payload"]["id"])',
133 api: [[2, 'typeddict-item'], [2, 'typeddict-item']],
134 },
135 {
136 id: 'sdk_result_assignment', code: 'count: int = composio.tools.get_raw_composio_tool_by_slug("X")',
137 api: [[1, 'assignment']],
138 },
139 { id: 'local_assignment', code: 'count: int = "one"', other: /\[assignment\]/ },
140 { id: 'sdk_binding_typo', code: 'sesion.tools()', api: [[1, 'name-defined']] },
141 { id: 'local_binding_typo', code: 'values = [1]\nprint(valuse)', other: /\[name-defined\]/ },
142 { id: 'bare_type_ignore', code: 'composio.tools.exectue("X", user_id="u", arguments={}) # type: ignore', directives: [[1, undefined]] },
143 {
144 id: 'unexplained_type_ignore', code: 'composio.tools.exectue("X", user_id="u", arguments={}) # type: ignore[attr-defined]',
145 directives: [[1, undefined]],
146 },
147 {
148 id: 'explained_type_ignore',
149 code: '# Fixture: a documented typing gap above the statement.\ncomposio.tools.exectue("X", user_id="u", arguments={}) # type: ignore[attr-defined]',
150 },
151 {
152 id: 'unused_type_ignore',
153 code: 'composio.tools.execute( # type: ignore[call-overload]\n # Fixture: the gap no longer exists.\n "X", user_id="u", arguments={}, dangerously_skip_version_check=True,\n)',
154 directives: [[1, 'unused-ignore']],
155 },
156 ];
157
158 const snippets: PythonSnippet[] = fixtures.map(({ id, code, typeCheck = true }) => ({
159 id, code, typeCheck, path: `fixtures/${id}.mdx`, line: 100,
160 }));
161 const result = runPythonHelper(snippets);
162 const violations = pythonSnippetViolations(snippets, result);
163 for (const fixture of fixtures) {
164 const actual = violations.filter(({ path }) => path === `fixtures/${fixture.id}.mdx`);
165 const expected = [
166 ...(fixture.api ?? []).map(([line, code]) => ({ line: 100 + line, rule: 'python-api', code })),
167 ...(fixture.syntax ?? []).map((line) => ({ line: 100 + line, rule: 'python-syntax', code: undefined })),
168 ...(fixture.directives ?? []).map(([line, code]) => ({ line: 100 + line, rule: 'python-directive', code })),
169 ];
170 assert.deepEqual(actual.map(({ line, rule, message }) => ({
171 line, rule, code: /\[([\w-]+)\]$/.exec(message)?.[1],
172 })), expected, `${fixture.id}: unexpected SDK/syntax attribution\n${JSON.stringify(result, null, 2)}`);
173 if (fixture.other) {
174 assert.ok(result.other.some(({ id, message }) => id === fixture.id && fixture.other!.test(message)),
175 `${fixture.id}: expected separate non-SDK diagnostic ${fixture.other}`);
176 }
177 }
178
179 // Syntax-only mode must not invoke mypy attribution, but still compile legacy code.
180 const syntaxOnly = runPythonHelper(snippets, true);
181 assert.deepEqual(syntaxOnly.api, []);
182 assert.deepEqual(syntaxOnly.other, []);
183 assert.deepEqual(syntaxOnly.directives, []);
184 assert.deepEqual(syntaxOnly.mypy, []);
185 assert.deepEqual(syntaxOnly.syntax.map(({ id, line }) => [id, line]), [['legacy_syntax', 1]]);
186 console.log(`Real Python helper regressions passed (${fixtures.length} fixtures; no snippets executed).`);