@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 014ece97c7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 07:15 UTC

[Frontend] Add tool filtering support to ToolServer (#29224)

Signed-off-by: Daniel Salib <danielsalib@meta.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>

daniel-salib committed Dec 1, 2025 at 03:03 UTC 014ece97c7aa49084a1119dca792af081a18dbc1
5 files changed +477 -25
examples/online_serving/openai_responses_client_with_mcp_tools.py new
+184
@@ -0,0 +1,184 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""
4 +Example demonstrating MCP (Model Context Protocol) tools with the Responses API.
5 +
6 +This example shows how to use MCP tools with different allowed_tools configurations:
7 +1. No filter (allows all tools from the MCP server)
8 +2. Wildcard "*" (explicitly allows all tools)
9 +3. Specific tool names (filters to only those tools)
10 +
11 +Set up this example by starting a vLLM OpenAI-compatible server with MCP tools enabled.
12 +For example:
13 +vllm serve openai/gpt-oss-20b --enforce-eager --tool-server demo
14 +
15 +Environment variables:
16 +- VLLM_ENABLE_RESPONSES_API_STORE=1
17 +- VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS=code_interpreter,container
18 +- VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS=1
19 +"""
20 +
21 +from openai import OpenAI
22 +from utils import get_first_model
23 +
24 +
25 +def example_no_filter():
26 + """Example with no allowed_tools filter - allows all tools."""
27 + print("=" * 60)
28 + print("Example 1: No allowed_tools filter (allows all tools)")
29 + print("=" * 60)
30 +
31 + base_url = "http://0.0.0.0:8000/v1"
32 + client = OpenAI(base_url=base_url, api_key="empty")
33 + model = get_first_model(client)
34 +
35 + response = client.responses.create(
36 + model=model,
37 + input="Execute this code: print('Hello from Python!')",
38 + instructions="Use the Python tool to execute code.",
39 + tools=[
40 + {
41 + "type": "mcp",
42 + "server_label": "code_interpreter",
43 + "server_url": "http://localhost:8888",
44 + # No allowed_tools specified - all tools are available
45 + }
46 + ],
47 + )
48 +
49 + print(f"Status: {response.status}")
50 + print(f"Output: {response.output_text}")
51 + print()
52 +
53 +
54 +def example_wildcard():
55 + """Example with allowed_tools=['*'] - explicitly allows all tools."""
56 + print("=" * 60)
57 + print("Example 2: allowed_tools=['*'] (select all tools)")
58 + print("=" * 60)
59 +
60 + base_url = "http://0.0.0.0:8000/v1"
61 + client = OpenAI(base_url=base_url, api_key="empty")
62 + model = get_first_model(client)
63 +
64 + response = client.responses.create(
65 + model=model,
66 + input="Execute this code: print('Hello from Python with wildcard!')",
67 + instructions="Use the Python tool to execute code.",
68 + tools=[
69 + {
70 + "type": "mcp",
71 + "server_label": "code_interpreter",
72 + "server_url": "http://localhost:8888",
73 + # Using "*" to explicitly allow all tools from this MCP server
74 + # This is equivalent to not specifying allowed_tools
75 + "allowed_tools": ["*"],
76 + }
77 + ],
78 + )
79 +
80 + print(f"Status: {response.status}")
81 + print(f"Output: {response.output_text}")
82 + print()
83 +
84 +
85 +def example_specific_tools():
86 + """Example with specific allowed_tools list - filters available tools.
87 +
88 + Note: This example uses 'web_search_preview' (browser) which has multiple
89 + sub-tools: 'search', 'open', 'find'. The code_interpreter (python) doesn't
90 + have sub-tools, so filtering doesn't apply there.
91 + """
92 + print("=" * 60)
93 + print("Example 3: allowed_tools=['search'] (filter browser to specific tools)")
94 + print("=" * 60)
95 +
96 + base_url = "http://0.0.0.0:8000/v1"
97 + client = OpenAI(base_url=base_url, api_key="empty")
98 + model = get_first_model(client)
99 +
100 + response = client.responses.create(
101 + model=model,
102 + input="Search for 'Python programming tutorials'",
103 + instructions="Use the browser tool to search.",
104 + tools=[
105 + {
106 + "type": "mcp",
107 + "server_label": "web_search_preview",
108 + "server_url": "http://localhost:8888",
109 + # Browser has tools: 'search', 'open', 'find'
110 + # Only allow 'search' - blocks 'open' and 'find'
111 + "allowed_tools": ["search"],
112 + }
113 + ],
114 + )
115 +
116 + print(f"Status: {response.status}")
117 + print(f"Output: {response.output_text}")
118 + print()
119 +
120 +
121 +def example_object_format():
122 + """Example using object format for allowed_tools with browser tools."""
123 + print("=" * 60)
124 + print("Example 4: allowed_tools with object format")
125 + print("=" * 60)
126 +
127 + base_url = "http://0.0.0.0:8000/v1"
128 + client = OpenAI(base_url=base_url, api_key="empty")
129 + model = get_first_model(client)
130 +
131 + response = client.responses.create(
132 + model=model,
133 + input="Search for 'machine learning' and open the first result",
134 + instructions="Use the browser tool.",
135 + tools=[
136 + {
137 + "type": "mcp",
138 + "server_label": "web_search_preview",
139 + "server_url": "http://localhost:8888",
140 + # Object format with tool_names field
141 + # Can also include read_only and other fields
142 + # Browser has tools: 'search', 'open', 'find'
143 + "allowed_tools": {
144 + "tool_names": [
145 + "search",
146 + "open",
147 + ], # Allow search and open, block find
148 + "read_only": False,
149 + },
150 + }
151 + ],
152 + )
153 +
154 + print(f"Status: {response.status}")
155 + print(f"Output: {response.output_text}")
156 + print()
157 +
158 +
159 +def main():
160 + """Run all examples."""
161 + print("\n" + "=" * 60)
162 + print("MCP Tools with allowed_tools Examples")
163 + print("=" * 60 + "\n")
164 +
165 + # Run all examples
166 + example_no_filter()
167 + example_wildcard()
168 + example_specific_tools()
169 + example_object_format()
170 +
171 + print("=" * 60)
172 + print("Summary:")
173 + print(" - No filter or '*' → All tools available from server")
174 + print(" - Specific list → Only those sub-tools available")
175 + print(" - Object format → More control with tool_names field")
176 + print("")
177 + print("Note: allowed_tools filters SUB-TOOLS within an MCP server:")
178 + print(" - code_interpreter (python): No sub-tools to filter")
179 + print(" - web_search_preview (browser): Has 'search', 'open', 'find'")
180 + print("=" * 60)
181 +
182 +
183 +if __name__ == "__main__":
184 + main()
tests/entrypoints/openai/test_response_api_mcp_tools.py
+100
@@ -4,6 +4,9 @@
4 import pytest
5 import pytest_asyncio
6 from openai import OpenAI
7 +from openai_harmony import ToolDescription, ToolNamespaceConfig
8 +
9 +from vllm.entrypoints.tool_server import MCPToolServer
10
11 from ...utils import RemoteOpenAIServer
12
@@ -111,6 +114,48 @@ async def test_mcp_tool_env_flag_enabled(mcp_enabled_client: OpenAI, model_name:
114 )
115
116
117 +@pytest.mark.asyncio
118 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
119 +async def test_mcp_tool_with_allowed_tools_star(
120 + mcp_enabled_client: OpenAI, model_name: str
121 +):
122 + """Test MCP tool with allowed_tools=['*'] to select all available tools.
123 +
124 + This E2E test verifies that the "*" wildcard works end-to-end.
125 + See test_serving_responses.py for detailed unit tests of "*" normalization.
126 + """
127 + response = await mcp_enabled_client.responses.create(
128 + model=model_name,
129 + input=(
130 + "Execute the following code: "
131 + "import random; print(random.randint(1, 1000000))"
132 + ),
133 + instructions=(
134 + "You must use the Python tool to execute code. Never simulate execution."
135 + ),
136 + tools=[
137 + {
138 + "type": "mcp",
139 + "server_label": "code_interpreter",
140 + "server_url": "http://localhost:8888",
141 + # Using "*" to allow all tools from this MCP server
142 + "allowed_tools": ["*"],
143 + }
144 + ],
145 + extra_body={"enable_response_messages": True},
146 + )
147 + assert response is not None
148 + assert response.status == "completed"
149 + # Verify tool calls work with allowed_tools=["*"]
150 + tool_call_found = False
151 + for message in response.output_messages:
152 + recipient = message.get("recipient")
153 + if recipient and recipient.startswith("python"):
154 + tool_call_found = True
155 + break
156 + assert tool_call_found, "Should have found at least one Python tool call with '*'"
157 +
158 +
159 @pytest.mark.asyncio
160 @pytest.mark.parametrize("model_name", [MODEL_NAME])
161 async def test_mcp_tool_env_flag_disabled(mcp_disabled_client: OpenAI, model_name: str):
@@ -159,3 +204,58 @@ async def test_mcp_tool_env_flag_disabled(mcp_disabled_client: OpenAI, model_nam
204 assert message.get("author").get("role") != "developer", (
205 "No developer messages should be present without a valid tool"
206 )
207 +
208 +
209 +def test_get_tool_description():
210 + """Test MCPToolServer.get_tool_description filtering logic.
211 +
212 + Note: The wildcard "*" is normalized to None by
213 + _extract_allowed_tools_from_mcp_requests before reaching this layer,
214 + so we only test None and specific tool filtering here.
215 + See test_serving_responses.py for "*" normalization tests.
216 + """
217 + pytest.importorskip("mcp")
218 +
219 + server = MCPToolServer()
220 + tool1 = ToolDescription.new(
221 + name="tool1", description="First", parameters={"type": "object"}
222 + )
223 + tool2 = ToolDescription.new(
224 + name="tool2", description="Second", parameters={"type": "object"}
225 + )
226 + tool3 = ToolDescription.new(
227 + name="tool3", description="Third", parameters={"type": "object"}
228 + )
229 +
230 + server.harmony_tool_descriptions = {
231 + "test_server": ToolNamespaceConfig(
232 + name="test_server", description="test", tools=[tool1, tool2, tool3]
233 + )
234 + }
235 +
236 + # Nonexistent server
237 + assert server.get_tool_description("nonexistent") is None
238 +
239 + # None (no filter) - returns all tools
240 + result = server.get_tool_description("test_server", allowed_tools=None)
241 + assert len(result.tools) == 3
242 +
243 + # Filter to specific tools
244 + result = server.get_tool_description(
245 + "test_server", allowed_tools=["tool1", "tool3"]
246 + )
247 + assert len(result.tools) == 2
248 + assert result.tools[0].name == "tool1"
249 + assert result.tools[1].name == "tool3"
250 +
251 + # Single tool
252 + result = server.get_tool_description("test_server", allowed_tools=["tool2"])
253 + assert len(result.tools) == 1
254 + assert result.tools[0].name == "tool2"
255 +
256 + # No matching tools - returns None
257 + result = server.get_tool_description("test_server", allowed_tools=["nonexistent"])
258 + assert result is None
259 +
260 + # Empty list - returns None
261 + assert server.get_tool_description("test_server", allowed_tools=[]) is None
tests/entrypoints/openai/test_serving_responses.py
+96
@@ -17,6 +17,7 @@ from vllm.entrypoints.context import ConversationContext
17 from vllm.entrypoints.openai.protocol import ErrorResponse, ResponsesRequest
18 from vllm.entrypoints.openai.serving_responses import (
19 OpenAIServingResponses,
20 + _extract_allowed_tools_from_mcp_requests,
21 extract_tool_types,
22 )
23 from vllm.entrypoints.tool_server import ToolServer
@@ -254,3 +255,98 @@ class TestValidateGeneratorInput:
255 # Should return an ErrorResponse
256 assert result is not None
257 assert isinstance(result, ErrorResponse)
258 +
259 +
260 +class TestExtractAllowedToolsFromMcpRequests:
261 + """Test class for _extract_allowed_tools_from_mcp_requests function"""
262 +
263 + def test_extract_allowed_tools_basic_formats(self):
264 + """Test extraction with list format, object format, and None."""
265 + from openai.types.responses.tool import McpAllowedToolsMcpToolFilter
266 +
267 + tools = [
268 + # List format
269 + Mcp(
270 + type="mcp",
271 + server_label="server1",
272 + allowed_tools=["tool1", "tool2"],
273 + ),
274 + # Object format
275 + Mcp(
276 + type="mcp",
277 + server_label="server2",
278 + allowed_tools=McpAllowedToolsMcpToolFilter(
279 + tool_names=["tool3", "tool4"]
280 + ),
281 + ),
282 + # None (no filter)
283 + Mcp(
284 + type="mcp",
285 + server_label="server3",
286 + allowed_tools=None,
287 + ),
288 + ]
289 + result = _extract_allowed_tools_from_mcp_requests(tools)
290 + assert result == {
291 + "server1": ["tool1", "tool2"],
292 + "server2": ["tool3", "tool4"],
293 + "server3": None,
294 + }
295 +
296 + def test_extract_allowed_tools_star_normalization(self):
297 + """Test that '*' wildcard is normalized to None (select all tools).
298 +
299 + This is the key test requested by reviewers to explicitly demonstrate
300 + that the "*" select-all scenario is handled correctly.
301 + """
302 + from openai.types.responses.tool import McpAllowedToolsMcpToolFilter
303 +
304 + tools = [
305 + # Star in list format
306 + Mcp(
307 + type="mcp",
308 + server_label="server1",
309 + allowed_tools=["*"],
310 + ),
311 + # Star mixed with other tools in list
312 + Mcp(
313 + type="mcp",
314 + server_label="server2",
315 + allowed_tools=["tool1", "*"],
316 + ),
317 + # Star in object format
318 + Mcp(
319 + type="mcp",
320 + server_label="server3",
321 + allowed_tools=McpAllowedToolsMcpToolFilter(tool_names=["*"]),
322 + ),
323 + ]
324 + result = _extract_allowed_tools_from_mcp_requests(tools)
325 + # All should be normalized to None (allows all tools)
326 + assert result == {
327 + "server1": None,
328 + "server2": None,
329 + "server3": None,
330 + }
331 +
332 + def test_extract_allowed_tools_filters_non_mcp(self):
333 + """Test that non-MCP tools are ignored during extraction."""
334 + tools = [
335 + Mcp(
336 + type="mcp",
337 + server_label="server1",
338 + allowed_tools=["tool1"],
339 + ),
340 + LocalShell(type="local_shell"), # Non-MCP tool should be ignored
341 + Mcp(
342 + type="mcp",
343 + server_label="server2",
344 + allowed_tools=["tool2"],
345 + ),
346 + ]
347 + result = _extract_allowed_tools_from_mcp_requests(tools)
348 + # Non-MCP tools should be ignored
349 + assert result == {
350 + "server1": ["tool1"],
351 + "server2": ["tool2"],
352 + }
vllm/entrypoints/openai/serving_responses.py
+68 -21
@@ -48,6 +48,7 @@ from openai.types.responses.response_output_text import Logprob, LogprobTopLogpr
48 from openai.types.responses.response_reasoning_item import (
49 Content as ResponseReasoningTextContent,
50 )
51 +from openai.types.responses.tool import Mcp, Tool
52 from openai_harmony import Message as OpenAIHarmonyMessage
53
54 from vllm import envs
@@ -111,6 +112,45 @@ from vllm.utils import random_uuid
112 logger = init_logger(__name__)
113
114
115 +def _extract_allowed_tools_from_mcp_requests(
116 + tools: list[Tool],
117 +) -> dict[str, list[str] | None]:
118 + """
119 + Extract allowed_tools mapping from MCP tool requests.
120 +
121 + Returns a dictionary mapping server_label to allowed_tools list.
122 + Handles both list format and McpAllowedToolsMcpToolFilter object format.
123 +
124 + Special handling:
125 + - If allowed_tools is None, returns None (allows all tools)
126 + - If allowed_tools contains "*", returns None (allows all tools)
127 + - Otherwise, returns the list of specific tool names
128 +
129 + This function can be reused for both harmony and non-harmony MCP calls.
130 + """
131 + allowed_tools_map: dict[str, list[str] | None] = {}
132 + for tool in tools:
133 + if not isinstance(tool, Mcp):
134 + continue
135 +
136 + # allowed_tools can be a list or an object with tool_names
137 + # Extract the actual list of tool names
138 + allowed_tools_val = None
139 + if tool.allowed_tools is not None:
140 + if isinstance(tool.allowed_tools, list):
141 + allowed_tools_val = tool.allowed_tools
142 + elif hasattr(tool.allowed_tools, "tool_names"):
143 + # It's an McpAllowedToolsMcpToolFilter object
144 + allowed_tools_val = tool.allowed_tools.tool_names
145 +
146 + # Normalize "*" to None (both mean "allow all tools")
147 + if allowed_tools_val is not None and "*" in allowed_tools_val:
148 + allowed_tools_val = None
149 +
150 + allowed_tools_map[tool.server_label] = allowed_tools_val
151 + return allowed_tools_map
152 +
153 +
154 class OpenAIServingResponses(OpenAIServing):
155 def __init__(
156 self,
@@ -878,38 +918,45 @@ class OpenAIServingResponses(OpenAIServing):
918 self, request: ResponsesRequest, with_custom_tools: bool, tool_types: set[str]
919 ) -> OpenAIHarmonyMessage:
920 reasoning_effort = request.reasoning.effort if request.reasoning else None
881 - enable_browser = (
882 - "web_search_preview" in tool_types
921 +
922 + # Extract allowed_tools from MCP tool requests
923 + allowed_tools_map = _extract_allowed_tools_from_mcp_requests(request.tools)
924 +
925 + # Get filtered tool descriptions first.
926 + # If get_tool_description returns None (due to filtering), the tool is disabled.
927 + browser_description = (
928 + self.tool_server.get_tool_description(
929 + "browser", allowed_tools_map.get("web_search_preview")
930 + )
931 + if "web_search_preview" in tool_types
932 and self.tool_server is not None
933 and self.tool_server.has_tool("browser")
934 + else None
935 )
886 - enable_code_interpreter = (
887 - "code_interpreter" in tool_types
936 + python_description = (
937 + self.tool_server.get_tool_description(
938 + "python", allowed_tools_map.get("code_interpreter")
939 + )
940 + if "code_interpreter" in tool_types
941 and self.tool_server is not None
942 and self.tool_server.has_tool("python")
943 + else None
944 )
891 - enable_container = (
892 - "container" in tool_types
945 + container_description = (
946 + self.tool_server.get_tool_description(
947 + "container", allowed_tools_map.get("container")
948 + )
949 + if "container" in tool_types
950 and self.tool_server is not None
951 and self.tool_server.has_tool("container")
952 + else None
953 )
954 +
955 sys_msg = get_system_message(
956 reasoning_effort=reasoning_effort,
898 - browser_description=(
899 - self.tool_server.get_tool_description("browser")
900 - if enable_browser and self.tool_server is not None
901 - else None
902 - ),
903 - python_description=(
904 - self.tool_server.get_tool_description("python")
905 - if enable_code_interpreter and self.tool_server is not None
906 - else None
907 - ),
908 - container_description=(
909 - self.tool_server.get_tool_description("container")
910 - if enable_container and self.tool_server is not None
911 - else None
912 - ),
957 + browser_description=browser_description,
958 + python_description=python_description,
959 + container_description=container_description,
960 instructions=request.instructions,
961 with_custom_tools=with_custom_tools,
962 )
vllm/entrypoints/tool_server.py
+29 -4
@@ -80,7 +80,9 @@ class ToolServer(ABC):
80 pass
81
82 @abstractmethod
83 - def get_tool_description(self, tool_name: str) -> ToolNamespaceConfig | None:
83 + def get_tool_description(
84 + self, tool_name: str, allowed_tools: list[str] | None = None
85 + ) -> ToolNamespaceConfig | None:
86 """
87 Return the tool description for the given tool name.
88 If the tool is not supported, return None.
@@ -147,8 +149,29 @@ class MCPToolServer(ToolServer):
149 def has_tool(self, tool_name: str):
150 return tool_name in self.harmony_tool_descriptions
151
150 - def get_tool_description(self, tool_name: str):
151 - return self.harmony_tool_descriptions.get(tool_name)
152 + def get_tool_description(
153 + self,
154 + server_label: str,
155 + allowed_tools: list[str] | None = None,
156 + ) -> ToolNamespaceConfig | None:
157 + cfg = self.harmony_tool_descriptions.get(server_label)
158 + if cfg is None:
159 + return None
160 +
161 + # No restrictions: all tools from this MCP server
162 + if allowed_tools is None:
163 + return cfg
164 +
165 + filtered = [t for t in cfg.tools if t.name in allowed_tools]
166 +
167 + if not filtered:
168 + return None
169 +
170 + return ToolNamespaceConfig(
171 + name=cfg.name,
172 + description=cfg.description,
173 + tools=filtered,
174 + )
175
176 @asynccontextmanager
177 async def new_session(
@@ -190,7 +213,9 @@ class DemoToolServer(ToolServer):
213 def has_tool(self, tool_name: str) -> bool:
214 return tool_name in self.tools
215
193 - def get_tool_description(self, tool_name: str) -> ToolNamespaceConfig | None:
216 + def get_tool_description(
217 + self, tool_name: str, allowed_tools: list[str] | None = None
218 + ) -> ToolNamespaceConfig | None:
219 if tool_name not in self.tools:
220 return None
221 if tool_name == "browser":