@devries-cameron20 / composio-1 / commits / 2415afba8
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

test(docs): add No Auth migration helper regressions

Cover the dry-run default, retired-toolkit rejection, metadata gates and the single POST without retries after ambiguous creation failures, plus 2,700 randomized cases. HTTP is mocked. Run with `bun run test:migration-helper`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Shashwat [zen] committed Oct 28, 2026 at 17:33 UTC 2415afba8d2e39e2288712805e03715dbdf466f8
3 files changed +160
docs/AGENTS.md
+1
@@ -35,6 +35,7 @@ bun run check:code-examples
35 bun run check:api-snippets
36 bun run check:python-snippets
37 bun run test:python-snippets
38 +bun run test:migration-helper
39 bun run check:typescript-snippets
40 bun run inventory:code-examples
41 ```
docs/package.json
+1
@@ -40,6 +40,7 @@
40 "check:api-snippets": "bun scripts/check-api-snippets.ts",
41 "check:python-snippets": "bun scripts/check-python-snippets.ts",
42 "test:python-snippets": "bun scripts/python-snippets/test-helper.ts",
43 + "test:migration-helper": "python3 -I tests/migrations/test_migrate_no_auth_to_instant.py",
44 "check:typescript-snippets": "bun scripts/check-typescript-snippets.ts",
45 "pretest": "bun run generate:kb",
46 "test": "bun test tests/static/",
docs/tests/migrations/test_migrate_no_auth_to_instant.py new
+158
@@ -0,0 +1,158 @@
1 +"""Offline regression and randomized tests for the downloadable No Auth migration helper.
2 +
3 +HTTP is mocked; no Composio or provider requests are made.
4 +Run from docs/: bun run test:migration-helper
5 +"""
6 +import contextlib, http.client, importlib.util, io, json, os, pathlib, random, socket, string, sys, unittest, urllib.error
7 +from unittest import mock
8 +
9 +sys.dont_write_bytecode = True
10 +PATH = pathlib.Path(__file__).resolve().parents[2] / "public/assets/migrations/migrate_no_auth_to_instant.py"
11 +spec = importlib.util.spec_from_file_location("helper", PATH)
12 +helper = importlib.util.module_from_spec(spec); spec.loader.exec_module(helper)
13 +
14 +
15 +class Resp(io.BytesIO):
16 + def __enter__(self): return self
17 + def __exit__(self, *a): self.close()
18 +
19 +
20 +def tool_meta(slug, toolkit, supported=True, price="$0.01 per call", discount=None):
21 + t = {"slug": slug, "toolkit": {"slug": toolkit}}
22 + if supported is not None:
23 + t["instant"] = {"supported": supported, "price": {"description": price} if price is not None else None}
24 + if discount is not None: t["instant"]["price"]["discount"] = discount
25 + return t
26 +
27 +
28 +class Fake:
29 + def __init__(self, metas, post):
30 + self.metas, self.post, self.calls = metas, post, []
31 + def __call__(self, req, timeout=None):
32 + self.calls.append((req.get_method(), req.full_url, req.data))
33 + assert req.headers.get("X-api-key") == "k"
34 + if req.get_method() == "GET":
35 + slug = req.full_url.split("/tools/")[1].split("?")[0]
36 + m = self.metas[slug]
37 + if isinstance(m, BaseException): raise m
38 + return Resp(json.dumps(m).encode())
39 + if isinstance(self.post, BaseException): raise self.post
40 + return Resp(json.dumps(self.post).encode() if not isinstance(self.post, bytes) else self.post)
41 + def posts(self): return [c for c in self.calls if c[0] == "POST"]
42 +
43 +
44 +def run(argv, fake=None, key="k"):
45 + out, err = io.StringIO(), io.StringIO()
46 + env = {"COMPOSIO_API_KEY": key} if key else {}
47 + opener = fake or mock.Mock(side_effect=AssertionError("network used"))
48 + with mock.patch.dict(os.environ, env, clear=True), mock.patch.object(helper.urllib.request, "urlopen", opener), \
49 + contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
50 + try: code = helper.main(argv)
51 + except SystemExit as e: code = e.code
52 + return code, out.getvalue(), err.getvalue()
53 +
54 +
55 +def http_err(code): return urllib.error.HTTPError("u", code, "x", {}, io.BytesIO(b""))
56 +
57 +BASE = ["--user-id", "user_123", "--tool", "exa:EXA_SEARCH"]
58 +
59 +
60 +class Regression(unittest.TestCase):
61 + def test_preview_default_no_network(self):
62 + code, out, _ = run(BASE)
63 + self.assertEqual(code, 0); self.assertIn("Preview only", out)
64 + body = json.loads(out[: out.rindex("}") + 1])["body"]
65 + self.assertEqual(body, {"user_id": "user_123", "toolkits": {"enable": ["exa"]}, "tools": {"exa": {"enable": ["EXA_SEARCH"]}},
66 + "instant": {"toolkits": {"enable": ["exa"]}, "tools": {"exa": {"enable": ["EXA_SEARCH"]}}}})
67 +
68 + def test_retired_toolkits_rejected(self):
69 + for tk in ["browser_tool", "composio_search", "text_to_pdf", "weathermap"]:
70 + code, _, err = run(["--user-id", "u", "--tool", f"{tk}:X_Y"])
71 + self.assertEqual(code, 2, tk); self.assertIn(tk, err)
72 +
73 + def test_text_to_pdf_points_to_pdf_co(self):
74 + _, _, err = run(["--user-id", "u", "--tool", "text_to_pdf:X"])
75 + self.assertIn("pdf_co", err)
76 +
77 + def test_bad_inputs(self):
78 + for t in ["exa", "exa:EXA:X", "EXA:EXA_SEARCH", "exa:exa_search", ":X", "exa:"]:
79 + self.assertEqual(run(["--user-id", "u", "--tool", t])[0], 2, t)
80 + self.assertEqual(run(BASE + ["--tool", "exa:EXA_SEARCH"])[0], 2)
81 + self.assertEqual(run(BASE + ["--tool", "tavily:EXA_SEARCH"])[0], 2)
82 + self.assertEqual(run(["--user-id", " ", "--tool", "exa:EXA_SEARCH"])[0], 2)
83 +
84 + def test_apply_requires_key(self):
85 + self.assertEqual(run(BASE + ["--apply"], key=None)[0], 2)
86 +
87 + def test_apply_happy_path(self):
88 + f = Fake({"EXA_SEARCH": tool_meta("EXA_SEARCH", "exa", discount="10%")}, {"session_id": "trs_1"})
89 + code, out, _ = run(BASE + ["--apply"], f)
90 + self.assertEqual(code, 0); self.assertIn("trs_1", out); self.assertIn("Published discount: 10%", out)
91 + self.assertEqual([c[0] for c in f.calls], ["GET", "POST"])
92 + self.assertIn("toolkit_versions=latest&include_pricing=true", f.calls[0][1])
93 +
94 + def test_post_failures_never_retry(self):
95 + for exc in [http_err(500), http_err(502), http_err(408), http_err(429), socket.timeout(), TimeoutError(),
96 + http.client.RemoteDisconnected(), ConnectionResetError(), http.client.IncompleteRead(b"")]:
97 + f = Fake({"EXA_SEARCH": tool_meta("EXA_SEARCH", "exa")}, exc)
98 + code, _, err = run(BASE + ["--apply"], f)
99 + self.assertEqual(code, 1, exc); self.assertEqual(len(f.posts()), 1, exc); self.assertIn("No automatic retry", err)
100 +
101 + def test_post_ambiguous_message_for_5xx(self):
102 + f = Fake({"EXA_SEARCH": tool_meta("EXA_SEARCH", "exa")}, http_err(503))
103 + self.assertIn("outcome is unknown", run(BASE + ["--apply"], f)[2])
104 +
105 + def test_post_bad_json_or_missing_id(self):
106 + for post in [b"not json", {"id": "x"}, [], {"session_id": ""}]:
107 + f = Fake({"EXA_SEARCH": tool_meta("EXA_SEARCH", "exa")}, post)
108 + code, _, _ = run(BASE + ["--apply"], f)
109 + self.assertEqual(code, 1, post); self.assertEqual(len(f.posts()), 1)
110 +
111 + def test_metadata_gates_block_creation(self):
112 + bad = [tool_meta("EXA_SEARCH", "exa", supported=None), tool_meta("EXA_SEARCH", "exa", supported=False),
113 + tool_meta("EXA_SEARCH", "tavily"), tool_meta("OTHER", "exa"), tool_meta("EXA_SEARCH", "exa", price=None),
114 + tool_meta("EXA_SEARCH", "exa", price=" "), [], {"slug": "EXA_SEARCH", "toolkit": "exa"},
115 + http_err(404), http_err(500), socket.timeout()]
116 + for m in bad:
117 + f = Fake({"EXA_SEARCH": m}, {"session_id": "trs_1"})
118 + self.assertEqual(run(BASE + ["--apply"], f)[0], 1, m); self.assertEqual(f.posts(), [], m)
119 +
120 +
121 +class Randomized(unittest.TestCase):
122 + N = 2700
123 +
124 + def test_random(self):
125 + rng = random.Random(20261113)
126 + chars = string.ascii_letters + string.digits + "_:-. "
127 + for i in range(self.N):
128 + n = rng.randint(1, 4); tools = []
129 + for _ in range(n):
130 + if rng.random() < 0.6:
131 + tk = rng.choice(["exa", "tavily", "serpapi", "browser_use", "pdf_co", "accu_weather", "gemini", "yelp", "seat_geek", "browser_tool", "weathermap"])
132 + tools.append(f"{tk}:{tk.upper()}_{rng.choice(['SEARCH', 'GET', 'RUN'])}")
133 + else:
134 + tools.append("".join(rng.choice(chars) for _ in range(rng.randint(0, 12))))
135 + argv = ["--user-id", "u"] + sum((["--tool", t] for t in tools), [])
136 + outcome = rng.choice(["ok", "unsupported", "post_err", "get_err", "no_id"])
137 + metas = {}
138 + for t in tools:
139 + if ":" in t:
140 + tk, slug = t.split(":", 1)
141 + metas[slug] = http_err(500) if outcome == "get_err" else tool_meta(slug, tk, supported=outcome != "unsupported")
142 + post = {"ok": {"session_id": "trs"}, "no_id": {}}.get(outcome, rng.choice([http_err(500), socket.timeout(), http_err(429)]))
143 + apply = rng.random() < 0.5
144 + f = Fake(metas, post)
145 + code, out, _ = run(argv + (["--apply"] if apply else []), f if apply else None)
146 + self.assertLessEqual(len(f.posts()), 1)
147 + if not apply: self.assertEqual(f.calls, [])
148 + if code == 0 and apply: self.assertEqual(outcome, "ok"); self.assertIn("trs", out)
149 + if outcome in ("unsupported", "get_err") and apply: self.assertEqual(f.posts(), [])
150 + if code == 0:
151 + body = json.loads(out[: out.index("\n}") + 2])["body"]
152 + self.assertEqual(body["toolkits"]["enable"], body["instant"]["toolkits"]["enable"])
153 + self.assertEqual(body["tools"], body["instant"]["tools"])
154 + self.assertFalse(set(body["tools"]) & set(helper.RETIRED))
155 +
156 +
157 +if __name__ == "__main__":
158 + unittest.main(verbosity=1)