Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
py 643 lines 24.8 KB
Raw
1 """
2 Comprehensive integration tests for Composio MCP functionality.
3
4 This module provides comprehensive pytest-based tests for the MCP (Model Context Protocol) functionality,
5 combining both structured pytest tests and direct execution tests for non-auth toolkits.
6
7 Usage:
8 export COMPOSIO_API_KEY="your_api_key_here"
9 pytest python/composio/integration_test/test_mcp.py -v
10 """
11
12 import os
13 import uuid
14
15 import pytest
16 import requests
17
18 from composio import Composio
19 from composio.exceptions import ValidationError
20
21 # Test configuration
22 API_KEY = os.getenv("COMPOSIO_API_KEY")
23 TEST_CONFIG_PREFIX = "pytest_integration_test"
24
25 if not API_KEY:
26 pytest.fail("COMPOSIO_API_KEY environment variable not set", pytrace=False)
27
28
29 def generate_unique_name(prefix: str = "pytest") -> str:
30 """Generate a unique test name using UUID to avoid collisions."""
31 # Use first 8 characters of UUID for readability while maintaining uniqueness
32 unique_id = str(uuid.uuid4())[:8]
33 return f"{prefix}-{unique_id}"
34
35
36 @pytest.fixture
37 def composio_client():
38 """Fixture providing Composio client instance."""
39 return Composio(api_key=API_KEY)
40
41
42 @pytest.fixture
43 def test_mcp_config_data():
44 """Fixture providing test data for MCP config creation."""
45 return {
46 "name": generate_unique_name("pytest-data"),
47 "toolkits": ["composio_search", "text_to_pdf"],
48 "allowed_tools": [
49 "COMPOSIO_SEARCH_DUCK_DUCK_GO_SEARCH",
50 "TEXT_TO_PDF_CONVERT_TEXT_TO_PDF",
51 ],
52 "manually_manage_connections": False,
53 }
54
55
56 class TestMCPStructure:
57 """Test the basic structure and availability of MCP features."""
58
59 def test_mcp_namespace_exists(self, composio_client):
60 """Test that mcp namespace exists at top level."""
61 assert hasattr(composio_client, "mcp"), "Missing mcp namespace"
62
63 @pytest.mark.parametrize(
64 "method_name", ["create", "list", "get", "update", "delete", "generate"]
65 )
66 def test_mcp_methods_available(self, composio_client, method_name):
67 """Test that all required MCP methods are available."""
68 assert hasattr(composio_client.mcp, method_name), (
69 f"Missing method: {method_name}"
70 )
71
72
73 class TestMCPOperations:
74 """Test MCP CRUD operations."""
75
76 def test_list_mcp_configs(self, composio_client):
77 """Test listing MCP configurations."""
78 try:
79 configs = composio_client.mcp.list()
80 assert isinstance(configs, dict), "list() should return a dictionary"
81 assert "items" in configs, "Response should contain 'items' key"
82 assert "current_page" in configs, (
83 "Response should contain 'current_page' key"
84 )
85 assert "total_pages" in configs, "Response should contain 'total_pages' key"
86 assert isinstance(configs["items"], list), "items should be a list"
87 except Exception as e:
88 # List might fail if no configs exist or API issues, but method should exist
89 assert "Failed to list MCP servers" in str(e) or "list" in str(e).lower()
90
91 def test_list_with_pagination(self, composio_client):
92 """Test listing with pagination parameters."""
93 try:
94 configs = composio_client.mcp.list(page_no=1, limit=5)
95 assert isinstance(configs, dict), (
96 "Paginated list should return a dictionary"
97 )
98 assert "items" in configs, "Response should contain 'items'"
99 except Exception as e:
100 # Expected to fail with current API implementation
101 assert "Failed to list MCP servers" in str(e)
102
103 def test_list_with_filters(self, composio_client):
104 """Test listing with filter parameters."""
105 try:
106 # Test toolkit filter with non-auth toolkits
107 configs_search = composio_client.mcp.list(toolkits="composio_search")
108 assert configs_search["items"] is None or isinstance(
109 configs_search["items"], list
110 )
111
112 # Test name filter
113 configs_name = composio_client.mcp.list(name="test")
114 assert configs_name["items"] is None or isinstance(
115 configs_name["items"], list
116 )
117 except Exception as e:
118 # Expected to fail with current API implementation
119 assert "Failed to list MCP servers" in str(e)
120
121 def test_create_mcp_config(self, composio_client):
122 """Test creating MCP configuration with new API using non-auth toolkits."""
123 # Server name must be ≤30 chars and only contain letters, numbers, spaces, hyphens
124 test_name = generate_unique_name("pytest-create")
125
126 mcp_config = composio_client.mcp.create(
127 test_name,
128 toolkits=["composio_search", "text_to_pdf"],
129 allowed_tools=[
130 "COMPOSIO_SEARCH_DUCK_DUCK_GO_SEARCH",
131 "TEXT_TO_PDF_CONVERT_TEXT_TO_PDF",
132 ],
133 manually_manage_connections=False,
134 )
135
136 # Basic response validation
137 assert mcp_config.id
138 assert mcp_config.name == test_name
139 assert callable(mcp_config.generate)
140 assert mcp_config.allowed_tools == [
141 "COMPOSIO_SEARCH_DUCK_DUCK_GO_SEARCH",
142 "TEXT_TO_PDF_CONVERT_TEXT_TO_PDF",
143 ]
144 assert mcp_config.commands.claude
145 assert mcp_config.commands.cursor
146 assert mcp_config.commands.windsurf
147
148 # Test the generate method
149 try:
150 server_instance = mcp_config.generate("test_user_123")
151 assert isinstance(server_instance, dict), (
152 "generate should return a dictionary"
153 )
154 assert "id" in server_instance, "Server instance should have id"
155 assert "url" in server_instance, "Server instance should have url"
156 assert "type" in server_instance, "Server instance should have type"
157 assert server_instance["type"] == "streamable_http", (
158 "Server type should be streamable_http"
159 )
160 assert isinstance(server_instance["url"], str), "URL should be string"
161 assert len(server_instance["url"]) > 0, "URL should not be empty"
162 except Exception as e:
163 print(f"Generate method failed (may be expected): {e}")
164
165 def test_get_nonexistent_config(self, composio_client):
166 """Test getting a non-existent configuration."""
167 with pytest.raises(ValidationError):
168 composio_client.mcp.get("nonexistent_config_id")
169
170 def test_create_with_empty_toolkits(self, composio_client):
171 """Test creating with empty toolkit configuration."""
172 with pytest.raises(ValidationError):
173 composio_client.mcp.create(
174 "test_empty",
175 toolkits=[], # Empty toolkits
176 )
177
178 def test_generate_method_directly(self, composio_client):
179 """Test the generate method directly on MCP class using non-auth toolkits."""
180 # First create a config
181 test_name = generate_unique_name("pytest-generate")
182
183 mcp_config = composio_client.mcp.create(
184 test_name,
185 toolkits=["composio_search", "text_to_pdf"],
186 allowed_tools=[
187 "COMPOSIO_SEARCH_DUCK_DUCK_GO_SEARCH",
188 "TEXT_TO_PDF_CONVERT_TEXT_TO_PDF",
189 ],
190 manually_manage_connections=False,
191 )
192
193 # Test generate method directly
194 try:
195 server_instance = composio_client.mcp.generate(
196 "test_user_direct_123",
197 mcp_config.id,
198 {"manually_manage_connections": False},
199 )
200
201 assert isinstance(server_instance, dict), (
202 "generate should return a dictionary"
203 )
204 assert server_instance["id"] == mcp_config.id, (
205 "Instance ID should match config ID"
206 )
207 assert server_instance["user_id"] == "test_user_direct_123", (
208 "User ID should match"
209 )
210 assert server_instance["type"] == "streamable_http", (
211 "Type should be streamable_http"
212 )
213 assert "url" in server_instance, "Should have URL"
214 except Exception as e:
215 print(f"Direct generate failed (may be expected): {e}")
216
217 def test_create_with_string_toolkits(self, composio_client):
218 """Test creating MCP configuration using simple string toolkit names."""
219 test_name = generate_unique_name("pytest-strings")
220
221 # Test with simple string toolkit names
222 mcp_config = composio_client.mcp.create(
223 test_name,
224 toolkits=["composio_search", "text_to_pdf"],
225 manually_manage_connections=False,
226 )
227
228 # Basic validation
229 assert mcp_config.id
230 assert mcp_config.name == test_name
231 assert callable(mcp_config.generate)
232
233 def test_create_with_mixed_toolkits(self, composio_client):
234 """Test creating MCP configuration with mixed string and object formats."""
235 test_name = generate_unique_name("pytest-mixed")
236
237 # Test with mixed formats (string and object with auth_config_id)
238 mcp_config = composio_client.mcp.create(
239 test_name,
240 toolkits=[
241 "composio_search", # String format
242 {
243 "toolkit": "text_to_pdf",
244 # No auth_config_id needed for non-auth toolkit
245 }, # Object format
246 ],
247 allowed_tools=[
248 "COMPOSIO_SEARCH_DUCK_DUCK_GO_SEARCH",
249 "TEXT_TO_PDF_CONVERT_TEXT_TO_PDF",
250 ],
251 manually_manage_connections=False,
252 )
253
254 # Basic validation
255 assert mcp_config.id
256 assert mcp_config.name == test_name
257 assert callable(mcp_config.generate)
258
259 def test_create_response_structure(self, composio_client):
260 """Test that create response has all required fields."""
261 test_name = generate_unique_name("pytest-structure")
262
263 mcp_config = composio_client.mcp.create(
264 test_name,
265 toolkits=["composio_search", "text_to_pdf"],
266 allowed_tools=[
267 "COMPOSIO_SEARCH_DUCK_DUCK_GO_SEARCH",
268 "TEXT_TO_PDF_CONVERT_TEXT_TO_PDF",
269 ],
270 manually_manage_connections=False,
271 )
272
273 # Basic structure validation
274 assert mcp_config.id
275 assert mcp_config.name == test_name
276 assert mcp_config.allowed_tools == [
277 "COMPOSIO_SEARCH_DUCK_DUCK_GO_SEARCH",
278 "TEXT_TO_PDF_CONVERT_TEXT_TO_PDF",
279 ]
280 assert mcp_config.auth_config_ids == []
281 assert mcp_config.mcp_url
282 assert mcp_config.commands.claude
283 assert mcp_config.commands.cursor
284 assert mcp_config.commands.windsurf
285 assert callable(mcp_config.generate)
286
287
288 class TestMCPErrorHandling:
289 """Test error handling and edge cases."""
290
291 @pytest.mark.parametrize(
292 "invalid_config_id", ["", "invalid_id", "mcp_000000", "nonexistent"]
293 )
294 def test_invalid_config_ids(self, composio_client, invalid_config_id):
295 """Test various invalid configuration IDs."""
296 with pytest.raises(ValidationError):
297 composio_client.mcp.get(invalid_config_id)
298
299 def test_generate_with_invalid_params(self, composio_client):
300 """Test generate method with invalid parameters."""
301 with pytest.raises(ValidationError):
302 composio_client.mcp.generate("", "invalid_config_id")
303
304 def test_create_with_invalid_toolkit_config(self, composio_client):
305 """Test create with invalid toolkit configuration."""
306 # Test with empty toolkit config - should fail during API call
307 try:
308 result = composio_client.mcp.create(
309 "test",
310 toolkits=[
311 {
312 # Empty toolkit config - will fail at API level
313 }
314 ],
315 )
316 # If it somehow succeeds, that's unexpected but not necessarily wrong
317 print(f"Create succeeded with empty config: {result.id}")
318 except ValidationError as e:
319 # Expected - should fail with validation error
320 assert "Failed to create MCP server" in str(e)
321 except Exception as e:
322 # Also acceptable - might fail for other API reasons
323 print(f"Create failed with: {type(e).__name__}: {e}")
324
325
326 class TestMCPNoAuthToolkits:
327 """Test MCP functionality with non-authentication toolkits."""
328
329 def test_mcp_with_no_auth_toolkits(self, composio_client):
330 """Test MCP with toolkits that don't require authentication."""
331 print("🔧 Testing MCP with Non-Auth Toolkits")
332 print("=" * 50)
333
334 # Create MCP server with non-auth toolkits
335 server_name = generate_unique_name("no-auth-test")
336 print(f"🚀 Creating MCP server: {server_name}")
337
338 mcp_server = composio_client.mcp.create(
339 server_name,
340 toolkits=["composio_search", "text_to_pdf"],
341 allowed_tools=[
342 "COMPOSIO_SEARCH_DUCK_DUCK_GO_SEARCH",
343 "TEXT_TO_PDF_CONVERT_TEXT_TO_PDF",
344 ],
345 manually_manage_connections=False,
346 )
347
348 print("✅ MCP server created successfully!")
349 print(f" Server ID: {mcp_server.id}")
350 print(f" Server Name: {mcp_server.name}")
351 print(f" Toolkits: {getattr(mcp_server, 'toolkits', 'N/A')}")
352
353 # Generate server instance for a test user
354 test_user_id = "test_user_no_auth_123"
355 print(f"\n🔗 Generating server instance for user: {test_user_id}")
356
357 server_instance = mcp_server.generate(test_user_id)
358
359 print("✅ Server instance generated successfully!")
360 print(f" Instance ID: {server_instance['id']}")
361 print(f" Instance Type: {server_instance['type']}")
362 print(f" Instance URL: {server_instance['url']}")
363 print(f" User ID: {server_instance['user_id']}")
364 print(f" Allowed Tools: {server_instance['allowed_tools']}")
365 print(f" Auth Configs: {server_instance['auth_configs']}")
366
367 # Test direct generate method as well
368 print("\n🔄 Testing direct generate method...")
369
370 direct_instance = composio_client.mcp.generate(
371 test_user_id + "_direct",
372 mcp_server.id,
373 {"manually_manage_connections": False},
374 )
375
376 print("✅ Direct generate method successful!")
377 print(f" Direct Instance URL: {direct_instance['url']}")
378 print(f" Direct Instance User ID: {direct_instance['user_id']}")
379
380 # Test URL connectivity (basic check)
381 print("\n🌐 Testing MCP URL connectivity...")
382
383 mcp_url = server_instance["url"]
384
385 try:
386 # Just verify the URL is valid and endpoint exists
387 # Don't try to read the stream as SSE endpoints can hang indefinitely
388 headers = {
389 "Accept": "text/event-stream, application/json, */*",
390 "Cache-Control": "no-cache",
391 "User-Agent": "Composio-Python-MCP-Test",
392 }
393
394 # Use HEAD request if supported, otherwise quick GET with immediate close
395 response = requests.head(mcp_url, headers=headers, timeout=3)
396
397 if response.status_code == 405: # Method not allowed for HEAD
398 # Try GET but immediately close without reading stream
399 response = requests.get(
400 mcp_url, headers=headers, timeout=3, stream=True
401 )
402 response.close() # Close immediately without reading
403
404 print("✅ MCP URL is accessible!")
405 print(f" Status Code: {response.status_code}")
406 print(f" URL: {mcp_url[:50]}...")
407
408 except requests.exceptions.Timeout:
409 print("⚠️ MCP URL timeout (normal for SSE endpoints)")
410 except Exception as e:
411 print(f"⚠️ MCP URL test failed: {e}")
412
413 print("\n📊 Test Summary:")
414 print(f" ✅ MCP Server Created: {mcp_server.id}")
415 print(f" ✅ Server Instance Generated: {server_instance['type']}")
416 print(" ✅ Direct Generate Method: Working")
417 print(f" ✅ Available Tools: {len(server_instance['allowed_tools'])} tools")
418 print(
419 f" ✅ No Auth Required: {len(server_instance['auth_configs'])} auth configs"
420 )
421
422 # Assertions for pytest
423 assert mcp_server.id is not None
424 assert mcp_server.name == server_name
425 assert server_instance["type"] == "streamable_http"
426 assert server_instance["user_id"] == test_user_id
427 assert len(server_instance["allowed_tools"]) > 0
428 assert len(server_instance["auth_configs"]) == 0 # No auth required
429
430 def test_mcp_with_string_toolkits(self, composio_client):
431 """Test MCP server creation using simple string toolkit names."""
432 print("🧪 Testing MCP with string toolkit names...")
433
434 # Create MCP server with string toolkit names (simplified API)
435 server_name = generate_unique_name("string-test")
436 print(f"🚀 Creating MCP server with strings: {server_name}")
437
438 mcp_server = composio_client.mcp.create(
439 server_name,
440 toolkits=["composio_search", "text_to_pdf"], # Simple strings
441 manually_manage_connections=False,
442 )
443
444 print("✅ MCP server created successfully with string toolkits!")
445 print(f" Server ID: {mcp_server.id}")
446 print(f" Server Name: {mcp_server.name}")
447
448 # Test generate method
449 user_id = f"string_user_{str(uuid.uuid4())[:8]}"
450 server_instance = mcp_server.generate(user_id)
451
452 print("✅ Server instance generated successfully!")
453 print(f" Instance URL: {server_instance['url']}")
454 print(f" User ID: {server_instance['user_id']}")
455 print(f" Server Type: {server_instance['type']}")
456
457 # Basic validation
458 assert server_instance["user_id"] == user_id
459 assert server_instance["type"] == "streamable_http"
460 assert "url" in server_instance
461 assert len(server_instance["url"]) > 0
462
463 print("🎉 String toolkit test completed successfully!")
464
465
466 class TestMCPRealWorldScenarios:
467 """Test real-world usage scenarios."""
468
469 def test_full_workflow_with_no_auth_toolkits(self, composio_client):
470 """Test complete workflow: create -> generate -> use with non-auth toolkits."""
471 # Server name must be ≤30 chars and only contain letters, numbers, spaces, hyphens
472 test_name = generate_unique_name("pytest-work")
473
474 # Step 1: Create MCP config
475 mcp_config = composio_client.mcp.create(
476 test_name,
477 toolkits=["composio_search", "text_to_pdf"],
478 allowed_tools=[
479 "COMPOSIO_SEARCH_DUCK_DUCK_GO_SEARCH",
480 "TEXT_TO_PDF_CONVERT_TEXT_TO_PDF",
481 ],
482 manually_manage_connections=False,
483 )
484
485 assert mcp_config.id is not None
486 assert mcp_config.name == test_name
487
488 # Step 2: Generate server instance
489 try:
490 server_instance = mcp_config.generate("workflow_user_123")
491
492 # Step 3: Verify server instance
493 assert server_instance["id"] == mcp_config.id
494 assert server_instance["user_id"] == "workflow_user_123"
495 assert server_instance["type"] == "streamable_http"
496 assert isinstance(server_instance["url"], str)
497 assert len(server_instance["url"]) > 0
498
499 print("✅ Full workflow successful!")
500 print(f" Config ID: {mcp_config.id}")
501 print(f" Server URL: {server_instance['url'][:50]}...")
502
503 except Exception as e:
504 print(f"Workflow generate step failed (may be expected): {e}")
505
506 def test_api_compatibility_with_typescript(self, composio_client):
507 """Test that Python API matches TypeScript patterns."""
508 # Check method availability (matching TypeScript)
509 mcp = composio_client.mcp
510
511 # CRUD operations
512 assert hasattr(mcp, "create"), "Missing create method"
513 assert hasattr(mcp, "list"), "Missing list method"
514 assert hasattr(mcp, "get"), "Missing get method"
515 assert hasattr(mcp, "update"), "Missing update method"
516 assert hasattr(mcp, "delete"), "Missing delete method"
517
518 # Instance generation
519 assert hasattr(mcp, "generate"), "Missing generate method"
520
521 print("✅ API compatibility verified with TypeScript patterns")
522
523 @pytest.mark.skip(
524 reason="MCP update bug with 'custom_tools' argument - TypeError in McpResource.update()"
525 )
526 def test_full_crud_cycle(self, composio_client):
527 """Test complete CRUD cycle: create -> get -> update -> get with assertions at each step."""
528 test_name = generate_unique_name("pytest-crud")
529
530 # Step 1: CREATE MCP server
531 print(f"🏗️ Step 1: Creating MCP server '{test_name}'")
532 mcp_server = composio_client.mcp.create(
533 test_name,
534 toolkits=["composio_search"],
535 allowed_tools=["COMPOSIO_SEARCH_DUCK_DUCK_GO_SEARCH"],
536 manually_manage_connections=False,
537 )
538
539 # Assertions after CREATE
540 assert mcp_server.id is not None, "Created server should have ID"
541 assert mcp_server.name == test_name, f"Server name should be {test_name}"
542 assert mcp_server.allowed_tools == ["COMPOSIO_SEARCH_DUCK_DUCK_GO_SEARCH"], (
543 "Should have correct allowed tools"
544 )
545 assert len(mcp_server.auth_config_ids) == 0, (
546 "Should have no auth configs for non-auth toolkits"
547 )
548 assert callable(mcp_server.generate), "Should have generate method"
549 print(f"✅ CREATE: Server created with ID {mcp_server.id}")
550
551 # Step 2: GET MCP server
552 print(f"📖 Step 2: Getting MCP server '{mcp_server.id}'")
553 retrieved_server = composio_client.mcp.get(mcp_server.id)
554
555 # Assertions after GET
556 assert retrieved_server.id == mcp_server.id, (
557 "Retrieved ID should match created ID"
558 )
559 assert retrieved_server.name == test_name, "Retrieved name should match"
560 print("✅ GET: Successfully retrieved server")
561
562 # Step 3: UPDATE MCP server
563 updated_name = f"{test_name}-updated"
564 print(
565 f"🔄 Step 3: Updating MCP server to '{updated_name}' with additional toolkit"
566 )
567 composio_client.mcp.update(
568 mcp_server.id,
569 name=updated_name,
570 toolkits=["composio_search", "text_to_pdf"],
571 allowed_tools=[
572 "COMPOSIO_SEARCH_DUCK_DUCK_GO_SEARCH",
573 "TEXT_TO_PDF_CONVERT_TEXT_TO_PDF",
574 ],
575 )
576 print("✅ UPDATE: Server updated successfully")
577
578 # Step 4: GET updated MCP server
579 print("📖 Step 4: Getting updated MCP server")
580 final_server = composio_client.mcp.get(mcp_server.id)
581
582 # Assertions after UPDATE and final GET
583 assert final_server.id == mcp_server.id, "ID should remain the same"
584
585 # Verify that updates took effect
586 assert final_server.name == updated_name, (
587 f"Name should be updated to {updated_name}"
588 )
589
590 # Check that toolkits were updated (should now include both composio_search and text_to_pdf)
591 expected_toolkits = ["composio_search", "text_to_pdf"]
592 actual_toolkits = getattr(final_server, "toolkits", [])
593 for toolkit in expected_toolkits:
594 assert toolkit in actual_toolkits, (
595 f"Updated toolkits should include {toolkit}"
596 )
597
598 # Check that allowed_tools were updated
599 expected_tools = [
600 "COMPOSIO_SEARCH_DUCK_DUCK_GO_SEARCH",
601 "TEXT_TO_PDF_CONVERT_TEXT_TO_PDF",
602 ]
603 actual_tools = getattr(final_server, "allowed_tools", [])
604 for tool in expected_tools:
605 assert tool in actual_tools, f"Updated allowed_tools should include {tool}"
606
607 print("✅ FINAL GET: Retrieved updated server with correct changes")
608 print(f" Updated name: {final_server.name}")
609 print(f" Updated toolkits: {actual_toolkits}")
610 print(f" Updated allowed_tools: {actual_tools}")
611
612 print(f"🎉 CRUD cycle test completed for server {mcp_server.id}")
613
614
615 # Direct execution support for backward compatibility
616 def main():
617 """Main test function for direct execution."""
618 print("🎯 MCP Integration Tests")
619 print("Testing MCP functionality with non-auth toolkits")
620 print()
621
622 # Initialize Composio client
623 composio = Composio()
624
625 # Run the no-auth toolkit test directly
626 try:
627 # Create test instance
628 test_instance = TestMCPNoAuthToolkits()
629 test_instance.test_mcp_with_no_auth_toolkits(composio)
630
631 print("\n🎉 All tests passed! MCP is working with non-auth toolkits.")
632 return True
633 except Exception as e:
634 print(f"\n💥 Tests failed: {e}")
635 import traceback
636
637 traceback.print_exc()
638 return False
639
640
641 if __name__ == "__main__":
642 success = main()
643 exit(0 if success else 1)