A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Frontend] Responses API MCP tools for built in tools and to pass through headers (#24628)
Signed-off-by: Alec Solder <alecs@fb.com> Signed-off-by: Alec S <10566873+alecsolder@users.noreply.github.com> Co-authored-by: Alec Solder <alecs@fb.com> Co-authored-by: Ye (Charlotte) Qi <yeq@meta.com>
Alec S committed
Sep 22, 2025 at 19:38 UTC
45d7d852d362b055b7c02eb967c4ea00d6f9e130
8 files changed
+463
-29
tests/entrypoints/openai/test_response_api_mcp_tools.py
new
+106
@@ -0,0 +1,106 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import pytest
5
+import pytest_asyncio
6
+from openai import OpenAI
7
+
8
+from ...utils import RemoteOpenAIServer
9
+
10
+MODEL_NAME = "openai/gpt-oss-20b"
11
+
12
+
13
+@pytest.fixture(scope="module")
14
+def monkeypatch_module():
15
+ from _pytest.monkeypatch import MonkeyPatch
16
+ mpatch = MonkeyPatch()
17
+ yield mpatch
18
+ mpatch.undo()
19
+
20
+
21
+@pytest.fixture(scope="module")
22
+def mcp_disabled_server(monkeypatch_module: pytest.MonkeyPatch):
23
+ args = ["--enforce-eager", "--tool-server", "demo"]
24
+
25
+ with monkeypatch_module.context() as m:
26
+ m.setenv("VLLM_ENABLE_RESPONSES_API_STORE", "1")
27
+ m.setenv("PYTHON_EXECUTION_BACKEND", "dangerously_use_uv")
28
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
29
+ yield remote_server
30
+
31
+
32
+@pytest.fixture(scope="function")
33
+def mcp_enabled_server(monkeypatch_module: pytest.MonkeyPatch):
34
+ args = ["--enforce-eager", "--tool-server", "demo"]
35
+
36
+ with monkeypatch_module.context() as m:
37
+ m.setenv("VLLM_ENABLE_RESPONSES_API_STORE", "1")
38
+ m.setenv("PYTHON_EXECUTION_BACKEND", "dangerously_use_uv")
39
+ m.setenv("GPT_OSS_SYSTEM_TOOL_MCP_LABELS",
40
+ "code_interpreter,container")
41
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
42
+ yield remote_server
43
+
44
+
45
+@pytest_asyncio.fixture
46
+async def mcp_disabled_client(mcp_disabled_server):
47
+ async with mcp_disabled_server.get_async_client() as async_client:
48
+ yield async_client
49
+
50
+
51
+@pytest_asyncio.fixture
52
+async def mcp_enabled_client(mcp_enabled_server):
53
+ async with mcp_enabled_server.get_async_client() as async_client:
54
+ yield async_client
55
+
56
+
57
+@pytest.mark.asyncio
58
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
59
+@pytest.mark.skip(reason="Code interpreter tool is not available in CI yet.")
60
+async def test_mcp_tool_env_flag_enabled(mcp_enabled_client: OpenAI,
61
+ model_name: str):
62
+ response = await mcp_enabled_client.responses.create(
63
+ model=model_name,
64
+ # TODO: Ideally should be able to set max tool calls
65
+ # to prevent multi-turn, but it is not currently supported
66
+ # would speed up the test
67
+ input=("What's the first 4 digits after the decimal point of "
68
+ "cube root of `19910212 * 20250910`? "
69
+ "Show only the digits. The python interpreter is not stateful "
70
+ "and you must print to see the output."),
71
+ tools=[{
72
+ "type": "mcp",
73
+ "server_label": "code_interpreter",
74
+ # URL unused for DemoToolServer
75
+ "server_url": "http://localhost:8888"
76
+ }],
77
+ )
78
+ assert response is not None
79
+ assert response.status == "completed"
80
+ assert response.usage.output_tokens_details.tool_output_tokens > 0
81
+
82
+
83
+@pytest.mark.asyncio
84
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
85
+@pytest.mark.skip(reason="Code interpreter tool is not available in CI yet.")
86
+async def test_mcp_tool_env_flag_disabled(mcp_disabled_client: OpenAI,
87
+ model_name: str):
88
+ response = await mcp_disabled_client.responses.create(
89
+ model=model_name,
90
+ # TODO: Ideally should be able to set max tool calls
91
+ # to prevent multi-turn, but it is not currently supported
92
+ # would speed up the test
93
+ input=("What's the first 4 digits after the decimal point of "
94
+ "cube root of `19910212 * 20250910`? "
95
+ "Show only the digits. The python interpreter is not stateful "
96
+ "and you must print to see the output."),
97
+ tools=[{
98
+ "type": "mcp",
99
+ "server_label": "code_interpreter",
100
+ # URL unused for DemoToolServer
101
+ "server_url": "http://localhost:8888"
102
+ }],
103
+ )
104
+ assert response is not None
105
+ assert response.status == "completed"
106
+ assert response.usage.output_tokens_details.tool_output_tokens == 0
tests/entrypoints/openai/test_response_api_with_harmony.py
+8
-1
@@ -454,7 +454,13 @@ async def test_web_search(client: OpenAI, model_name: str):
454
async def test_code_interpreter(client: OpenAI, model_name: str):
455
response = await client.responses.create(
456
model=model_name,
457
- input="Multiply 64548*15151 using builtin python interpreter.",
457
+ # TODO: Ideally should be able to set max tool calls
458
+ # to prevent multi-turn, but it is not currently supported
459
+ # would speed up the test
460
+ input=("What's the first 4 digits after the decimal point of "
461
+ "cube root of `19910212 * 20250910`? "
462
+ "Show only the digits. The python interpreter is not stateful "
463
+ "and you must print to see the output."),
464
tools=[{
465
"type": "code_interpreter",
466
"container": {
@@ -464,6 +470,7 @@ async def test_code_interpreter(client: OpenAI, model_name: str):
470
)
471
assert response is not None
472
assert response.status == "completed"
473
+ assert response.usage.output_tokens_details.tool_output_tokens > 0
474
475
476
def get_weather(latitude, longitude):
tests/test_envs.py
new
+216
@@ -0,0 +1,216 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import os
5
+from unittest.mock import patch
6
+
7
+import pytest
8
+
9
+from vllm.envs import env_list_with_choices, env_with_choices
10
+
11
+
12
+class TestEnvWithChoices:
13
+ """Test cases for env_with_choices function."""
14
+
15
+ def test_default_value_returned_when_env_not_set(self):
16
+ """Test default is returned when env var is not set."""
17
+ env_func = env_with_choices("NONEXISTENT_ENV", "default",
18
+ ["option1", "option2"])
19
+ assert env_func() == "default"
20
+
21
+ def test_none_default_returned_when_env_not_set(self):
22
+ """Test that None is returned when env not set and default is None."""
23
+ env_func = env_with_choices("NONEXISTENT_ENV", None,
24
+ ["option1", "option2"])
25
+ assert env_func() is None
26
+
27
+ def test_valid_value_returned_case_sensitive(self):
28
+ """Test that valid value is returned in case sensitive mode."""
29
+ with patch.dict(os.environ, {"TEST_ENV": "option1"}):
30
+ env_func = env_with_choices("TEST_ENV",
31
+ "default", ["option1", "option2"],
32
+ case_sensitive=True)
33
+ assert env_func() == "option1"
34
+
35
+ def test_valid_lowercase_value_returned_case_insensitive(self):
36
+ """Test that lowercase value is accepted in case insensitive mode."""
37
+ with patch.dict(os.environ, {"TEST_ENV": "option1"}):
38
+ env_func = env_with_choices("TEST_ENV",
39
+ "default", ["OPTION1", "OPTION2"],
40
+ case_sensitive=False)
41
+ assert env_func() == "option1"
42
+
43
+ def test_valid_uppercase_value_returned_case_insensitive(self):
44
+ """Test that uppercase value is accepted in case insensitive mode."""
45
+ with patch.dict(os.environ, {"TEST_ENV": "OPTION1"}):
46
+ env_func = env_with_choices("TEST_ENV",
47
+ "default", ["option1", "option2"],
48
+ case_sensitive=False)
49
+ assert env_func() == "OPTION1"
50
+
51
+ def test_invalid_value_raises_error_case_sensitive(self):
52
+ """Test that invalid value raises ValueError in case sensitive mode."""
53
+ with patch.dict(os.environ, {"TEST_ENV": "invalid"}):
54
+ env_func = env_with_choices("TEST_ENV",
55
+ "default", ["option1", "option2"],
56
+ case_sensitive=True)
57
+ with pytest.raises(ValueError,
58
+ match="Invalid value 'invalid' for TEST_ENV"):
59
+ env_func()
60
+
61
+ def test_case_mismatch_raises_error_case_sensitive(self):
62
+ """Test that case mismatch raises ValueError in case sensitive mode."""
63
+ with patch.dict(os.environ, {"TEST_ENV": "OPTION1"}):
64
+ env_func = env_with_choices("TEST_ENV",
65
+ "default", ["option1", "option2"],
66
+ case_sensitive=True)
67
+ with pytest.raises(ValueError,
68
+ match="Invalid value 'OPTION1' for TEST_ENV"):
69
+ env_func()
70
+
71
+ def test_invalid_value_raises_error_case_insensitive(self):
72
+ """Test that invalid value raises ValueError when case insensitive."""
73
+ with patch.dict(os.environ, {"TEST_ENV": "invalid"}):
74
+ env_func = env_with_choices("TEST_ENV",
75
+ "default", ["option1", "option2"],
76
+ case_sensitive=False)
77
+ with pytest.raises(ValueError,
78
+ match="Invalid value 'invalid' for TEST_ENV"):
79
+ env_func()
80
+
81
+ def test_callable_choices_resolved_correctly(self):
82
+ """Test that callable choices are resolved correctly."""
83
+
84
+ def get_choices():
85
+ return ["dynamic1", "dynamic2"]
86
+
87
+ with patch.dict(os.environ, {"TEST_ENV": "dynamic1"}):
88
+ env_func = env_with_choices("TEST_ENV", "default", get_choices)
89
+ assert env_func() == "dynamic1"
90
+
91
+ def test_callable_choices_with_invalid_value(self):
92
+ """Test that callable choices raise error for invalid values."""
93
+
94
+ def get_choices():
95
+ return ["dynamic1", "dynamic2"]
96
+
97
+ with patch.dict(os.environ, {"TEST_ENV": "invalid"}):
98
+ env_func = env_with_choices("TEST_ENV", "default", get_choices)
99
+ with pytest.raises(ValueError,
100
+ match="Invalid value 'invalid' for TEST_ENV"):
101
+ env_func()
102
+
103
+
104
+class TestEnvListWithChoices:
105
+ """Test cases for env_list_with_choices function."""
106
+
107
+ def test_default_list_returned_when_env_not_set(self):
108
+ """Test that default list is returned when env var is not set."""
109
+ env_func = env_list_with_choices("NONEXISTENT_ENV",
110
+ ["default1", "default2"],
111
+ ["option1", "option2"])
112
+ assert env_func() == ["default1", "default2"]
113
+
114
+ def test_empty_default_list_returned_when_env_not_set(self):
115
+ """Test that empty default list is returned when env not set."""
116
+ env_func = env_list_with_choices("NONEXISTENT_ENV", [],
117
+ ["option1", "option2"])
118
+ assert env_func() == []
119
+
120
+ def test_single_valid_value_parsed_correctly(self):
121
+ """Test that single valid value is parsed correctly."""
122
+ with patch.dict(os.environ, {"TEST_ENV": "option1"}):
123
+ env_func = env_list_with_choices("TEST_ENV", [],
124
+ ["option1", "option2"])
125
+ assert env_func() == ["option1"]
126
+
127
+ def test_multiple_valid_values_parsed_correctly(self):
128
+ """Test that multiple valid values are parsed correctly."""
129
+ with patch.dict(os.environ, {"TEST_ENV": "option1,option2"}):
130
+ env_func = env_list_with_choices("TEST_ENV", [],
131
+ ["option1", "option2"])
132
+ assert env_func() == ["option1", "option2"]
133
+
134
+ def test_values_with_whitespace_trimmed(self):
135
+ """Test that values with whitespace are trimmed correctly."""
136
+ with patch.dict(os.environ, {"TEST_ENV": " option1 , option2 "}):
137
+ env_func = env_list_with_choices("TEST_ENV", [],
138
+ ["option1", "option2"])
139
+ assert env_func() == ["option1", "option2"]
140
+
141
+ def test_empty_values_filtered_out(self):
142
+ """Test that empty values are filtered out."""
143
+ with patch.dict(os.environ, {"TEST_ENV": "option1,,option2,"}):
144
+ env_func = env_list_with_choices("TEST_ENV", [],
145
+ ["option1", "option2"])
146
+ assert env_func() == ["option1", "option2"]
147
+
148
+ def test_empty_string_returns_default(self):
149
+ """Test that empty string returns default."""
150
+ with patch.dict(os.environ, {"TEST_ENV": ""}):
151
+ env_func = env_list_with_choices("TEST_ENV", ["default"],
152
+ ["option1", "option2"])
153
+ assert env_func() == ["default"]
154
+
155
+ def test_only_commas_returns_default(self):
156
+ """Test that string with only commas returns default."""
157
+ with patch.dict(os.environ, {"TEST_ENV": ",,,"}):
158
+ env_func = env_list_with_choices("TEST_ENV", ["default"],
159
+ ["option1", "option2"])
160
+ assert env_func() == ["default"]
161
+
162
+ def test_case_sensitive_validation(self):
163
+ """Test case sensitive validation."""
164
+ with patch.dict(os.environ, {"TEST_ENV": "option1,OPTION2"}):
165
+ env_func = env_list_with_choices("TEST_ENV", [],
166
+ ["option1", "option2"],
167
+ case_sensitive=True)
168
+ with pytest.raises(ValueError,
169
+ match="Invalid value 'OPTION2' in TEST_ENV"):
170
+ env_func()
171
+
172
+ def test_case_insensitive_validation(self):
173
+ """Test case insensitive validation."""
174
+ with patch.dict(os.environ, {"TEST_ENV": "OPTION1,option2"}):
175
+ env_func = env_list_with_choices("TEST_ENV", [],
176
+ ["option1", "option2"],
177
+ case_sensitive=False)
178
+ assert env_func() == ["OPTION1", "option2"]
179
+
180
+ def test_invalid_value_in_list_raises_error(self):
181
+ """Test that invalid value in list raises ValueError."""
182
+ with patch.dict(os.environ, {"TEST_ENV": "option1,invalid,option2"}):
183
+ env_func = env_list_with_choices("TEST_ENV", [],
184
+ ["option1", "option2"])
185
+ with pytest.raises(ValueError,
186
+ match="Invalid value 'invalid' in TEST_ENV"):
187
+ env_func()
188
+
189
+ def test_callable_choices_resolved_correctly(self):
190
+ """Test that callable choices are resolved correctly."""
191
+
192
+ def get_choices():
193
+ return ["dynamic1", "dynamic2"]
194
+
195
+ with patch.dict(os.environ, {"TEST_ENV": "dynamic1,dynamic2"}):
196
+ env_func = env_list_with_choices("TEST_ENV", [], get_choices)
197
+ assert env_func() == ["dynamic1", "dynamic2"]
198
+
199
+ def test_callable_choices_with_invalid_value(self):
200
+ """Test that callable choices raise error for invalid values."""
201
+
202
+ def get_choices():
203
+ return ["dynamic1", "dynamic2"]
204
+
205
+ with patch.dict(os.environ, {"TEST_ENV": "dynamic1,invalid"}):
206
+ env_func = env_list_with_choices("TEST_ENV", [], get_choices)
207
+ with pytest.raises(ValueError,
208
+ match="Invalid value 'invalid' in TEST_ENV"):
209
+ env_func()
210
+
211
+ def test_duplicate_values_preserved(self):
212
+ """Test that duplicate values in the list are preserved."""
213
+ with patch.dict(os.environ, {"TEST_ENV": "option1,option1,option2"}):
214
+ env_func = env_list_with_choices("TEST_ENV", [],
215
+ ["option1", "option2"])
216
+ assert env_func() == ["option1", "option1", "option2"]
vllm/entrypoints/context.py
+30
-7
@@ -8,6 +8,7 @@ from abc import ABC, abstractmethod
8
from contextlib import AsyncExitStack
9
from typing import TYPE_CHECKING, Optional, Union
10
11
+from openai.types.responses.tool import Mcp
12
from openai_harmony import Author, Message, Role, StreamState, TextContent
13
14
from vllm.entrypoints.harmony_utils import (
@@ -21,6 +22,24 @@ if TYPE_CHECKING:
22
23
logger = logging.getLogger(__name__)
24
25
+# This is currently needed as the tool type doesn't 1:1 match the
26
+# tool namespace, which is what is used to look up the
27
+# connection to the tool server
28
+_TOOL_NAME_TO_TYPE_MAP = {
29
+ "browser": "web_search_preview",
30
+ "python": "code_interpreter",
31
+ "container": "container",
32
+}
33
+
34
+
35
+def _map_tool_name_to_tool_type(tool_name: str) -> str:
36
+ if tool_name not in _TOOL_NAME_TO_TYPE_MAP:
37
+ available_tools = ', '.join(_TOOL_NAME_TO_TYPE_MAP.keys())
38
+ raise ValueError(
39
+ f"Built-in tool name '{tool_name}' not defined in mapping. "
40
+ f"Available tools: {available_tools}")
41
+ return _TOOL_NAME_TO_TYPE_MAP[tool_name]
42
+
43
44
class TurnTokens:
45
"""Tracks token counts for a single conversation turn."""
@@ -59,8 +78,8 @@ class ConversationContext(ABC):
78
79
@abstractmethod
80
async def init_tool_sessions(self, tool_server: Optional[ToolServer],
62
- exit_stack: AsyncExitStack,
63
- request_id: str) -> None:
81
+ exit_stack: AsyncExitStack, request_id: str,
82
+ mcp_tools: dict[str, Mcp]) -> None:
83
pass
84
85
@abstractmethod
@@ -96,8 +115,8 @@ class SimpleContext(ConversationContext):
115
raise NotImplementedError("Should not be called.")
116
117
async def init_tool_sessions(self, tool_server: Optional[ToolServer],
99
- exit_stack: AsyncExitStack,
100
- request_id: str) -> None:
118
+ exit_stack: AsyncExitStack, request_id: str,
119
+ mcp_tools: dict[str, Mcp]) -> None:
120
pass
121
122
async def cleanup_session(self) -> None:
@@ -318,13 +337,17 @@ class HarmonyContext(ConversationContext):
337
]
338
339
async def init_tool_sessions(self, tool_server: Optional[ToolServer],
321
- exit_stack: AsyncExitStack,
322
- request_id: str) -> None:
340
+ exit_stack: AsyncExitStack, request_id: str,
341
+ mcp_tools: dict[str, Mcp]):
342
if tool_server:
343
for tool_name in self.available_tools:
344
if tool_name not in self._tool_sessions:
345
+ tool_type = _map_tool_name_to_tool_type(tool_name)
346
+ headers = mcp_tools[
347
+ tool_type].headers if tool_type in mcp_tools else None
348
tool_session = await exit_stack.enter_async_context(
327
- tool_server.new_session(tool_name, request_id))
349
+ tool_server.new_session(tool_name, request_id,
350
+ headers))
351
self._tool_sessions[tool_name] = tool_session
352
exit_stack.push_async_exit(self.cleanup_session)
353
vllm/entrypoints/harmony_utils.py
+3
-1
@@ -126,8 +126,10 @@ def get_developer_message(
126
function_tools: list[Union[Tool, ChatCompletionToolsParam]] = []
127
for tool in tools:
128
if tool.type in ("web_search_preview", "code_interpreter",
129
- "container"):
129
+ "container", "mcp"):
130
# These are built-in tools that are added to the system message.
131
+ # Adding in MCP for now until we support MCP tools executed
132
+ # server side
133
pass
134
135
elif tool.type == "function":
vllm/entrypoints/openai/serving_responses.py
+19
-6
@@ -460,8 +460,12 @@ class OpenAIServingResponses(OpenAIServing):
460
461
async with AsyncExitStack() as exit_stack:
462
try:
463
+ mcp_tools = {
464
+ tool.server_label: tool
465
+ for tool in request.tools if tool.type == "mcp"
466
+ }
467
await context.init_tool_sessions(self.tool_server, exit_stack,
464
- request.request_id)
468
+ request.request_id, mcp_tools)
469
async for _ in result_generator:
470
pass
471
except asyncio.CancelledError:
@@ -748,11 +752,16 @@ class OpenAIServingResponses(OpenAIServing):
752
# New conversation.
753
reasoning_effort = (request.reasoning.effort
754
if request.reasoning else None)
751
- # Temporary: OpenAI types doesn't have container tool
752
- # so we used MCP to cover that, up for change
755
tool_types = [tool.type for tool in request.tools]
754
- if envs.VLLM_GPT_OSS_USE_CONTAINER_TOOL:
755
- tool_types.append("container")
756
+
757
+ # Allow the MCP Tool type to enable built in tools if the
758
+ # server_label is allowlisted in
759
+ # envs.GPT_OSS_SYSTEM_TOOL_MCP_LABELS
760
+ if envs.GPT_OSS_SYSTEM_TOOL_MCP_LABELS:
761
+ for tool in request.tools:
762
+ if (tool.type == "mcp" and tool.server_label
763
+ in envs.GPT_OSS_SYSTEM_TOOL_MCP_LABELS):
764
+ tool_types.append(tool.server_label)
765
enable_browser = ("web_search_preview" in tool_types
766
and self.tool_server is not None
767
and self.tool_server.has_tool("browser"))
@@ -1653,8 +1662,12 @@ class OpenAIServingResponses(OpenAIServing):
1662
async with AsyncExitStack() as exit_stack:
1663
processer = None
1664
if self.use_harmony:
1665
+ mcp_tools = {
1666
+ tool.server_label: tool
1667
+ for tool in request.tools if tool.type == "mcp"
1668
+ }
1669
await context.init_tool_sessions(self.tool_server, exit_stack,
1657
- request.request_id)
1670
+ request.request_id, mcp_tools)
1671
processer = self._process_harmony_streaming_events
1672
else:
1673
processer = self._process_simple_streaming_events
vllm/entrypoints/tool_server.py
+20
-9
@@ -18,7 +18,6 @@ if TYPE_CHECKING:
18
async def list_server_and_tools(server_url: str):
19
from mcp import ClientSession
20
from mcp.client.sse import sse_client
21
-
21
async with sse_client(url=server_url) as streams, ClientSession(
22
*streams) as session:
23
initialize_response = await session.initialize()
@@ -86,8 +85,12 @@ class ToolServer(ABC):
85
pass
86
87
@abstractmethod
89
- def new_session(self, tool_name: str,
90
- session_id: str) -> AbstractAsyncContextManager[Any]:
88
+ def new_session(
89
+ self,
90
+ tool_name: str,
91
+ session_id: str,
92
+ headers: Optional[dict[str, str]] = None
93
+ ) -> AbstractAsyncContextManager[Any]:
94
"""
95
Create a session for the tool.
96
"""
@@ -144,16 +147,21 @@ class MCPToolServer(ToolServer):
147
return self.harmony_tool_descriptions.get(tool_name)
148
149
@asynccontextmanager
147
- async def new_session(self, tool_name: str, session_id: str):
150
+ async def new_session(self,
151
+ tool_name: str,
152
+ session_id: str,
153
+ headers: Optional[dict[str, str]] = None):
154
from mcp import ClientSession
155
from mcp.client.sse import sse_client
156
url = self.urls.get(tool_name)
151
- headers = {"x-session-id": session_id}
157
+ request_headers = {"x-session-id": session_id}
158
+ if headers is not None:
159
+ request_headers.update(headers)
160
if not url:
161
raise KeyError(f"Tool '{tool_name}' is not supported")
154
- async with sse_client(url=url,
155
- headers=headers) as streams, ClientSession(
156
- *streams) as session:
162
+ async with sse_client(
163
+ url=url, headers=request_headers) as streams, ClientSession(
164
+ *streams) as session:
165
await session.initialize()
166
yield session
167
@@ -189,7 +197,10 @@ class DemoToolServer(ToolServer):
197
raise ValueError(f"Unknown tool {tool_name}")
198
199
@asynccontextmanager
192
- async def new_session(self, tool_name: str, session_id: str):
200
+ async def new_session(self,
201
+ tool_name: str,
202
+ session_id: str,
203
+ headers: Optional[dict[str, str]] = None):
204
if tool_name not in self.tools:
205
raise KeyError(f"Tool '{tool_name}' is not supported")
206
yield self.tools[tool_name]
vllm/envs.py
+61
-5
@@ -185,11 +185,11 @@ if TYPE_CHECKING:
185
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = False
186
VLLM_TUNED_CONFIG_FOLDER: Optional[str] = None
187
VLLM_DISABLE_PAD_FOR_CUDAGRAPH: bool = False
188
- VLLM_GPT_OSS_USE_CONTAINER_TOOL: bool = False
188
VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: bool = False
189
VLLM_CUSTOM_SCOPES_FOR_PROFILING: bool = False
190
VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES: bool = True
191
VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME: str = "VLLM_OBJECT_STORAGE_SHM_BUFFER"
192
+ GPT_OSS_SYSTEM_TOOL_MCP_LABELS: list[str] = []
193
VLLM_PATTERN_MATCH_DEBUG: Optional[str] = None
194
195
@@ -261,6 +261,58 @@ def env_with_choices(
261
return _get_validated_env
262
263
264
+def env_list_with_choices(
265
+ env_name: str,
266
+ default: list[str],
267
+ choices: Union[list[str], Callable[[], list[str]]],
268
+ case_sensitive: bool = True) -> Callable[[], list[str]]:
269
+ """
270
+ Create a lambda that validates environment variable
271
+ containing comma-separated values against allowed choices
272
+
273
+ Args:
274
+ env_name: Name of the environment variable
275
+ default: Default list of values if not set
276
+ choices: List of valid string options or callable that returns list
277
+ case_sensitive: Whether validation should be case sensitive
278
+
279
+ Returns:
280
+ Lambda function for environment_variables
281
+ dict that returns list of strings
282
+ """
283
+
284
+ def _get_validated_env_list() -> list[str]:
285
+ value = os.getenv(env_name)
286
+ if value is None:
287
+ return default
288
+
289
+ # Split comma-separated values and strip whitespace
290
+ values = [v.strip() for v in value.split(",") if v.strip()]
291
+
292
+ if not values:
293
+ return default
294
+
295
+ # Resolve choices if it's a callable (for lazy loading)
296
+ actual_choices = choices() if callable(choices) else choices
297
+
298
+ # Validate each value
299
+ for val in values:
300
+ if not case_sensitive:
301
+ check_value = val.lower()
302
+ check_choices = [choice.lower() for choice in actual_choices]
303
+ else:
304
+ check_value = val
305
+ check_choices = actual_choices
306
+
307
+ if check_value not in check_choices:
308
+ raise ValueError(f"Invalid value '{val}' in {env_name}. "
309
+ f"Valid options: {actual_choices}.")
310
+
311
+ return values
312
+
313
+ return _get_validated_env_list
314
+
315
+
316
def get_vllm_port() -> Optional[int]:
317
"""Get the port from VLLM_PORT environment variable.
318
@@ -1320,10 +1372,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
1372
"VLLM_TUNED_CONFIG_FOLDER":
1373
lambda: os.getenv("VLLM_TUNED_CONFIG_FOLDER", None),
1374
1323
- # Allows vllm use container tool
1324
- "VLLM_GPT_OSS_USE_CONTAINER_TOOL":
1325
- lambda: bool(int(os.getenv("VLLM_GPT_OSS_USE_CONTAINER_TOOL", "0"))),
1326
-
1375
# Allows harmony instructions to be injected on system messages
1376
"VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS":
1377
lambda: bool(
@@ -1343,6 +1391,14 @@ environment_variables: dict[str, Callable[[], Any]] = {
1391
"VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME":
1392
lambda: os.getenv("VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME",
1393
"VLLM_OBJECT_STORAGE_SHM_BUFFER"),
1394
+
1395
+ # Valid values are container,code_interpreter,web_search_preview
1396
+ # ex GPT_OSS_SYSTEM_TOOL_MCP_LABELS=container,code_interpreter
1397
+ "GPT_OSS_SYSTEM_TOOL_MCP_LABELS":
1398
+ env_list_with_choices("GPT_OSS_SYSTEM_TOOL_MCP_LABELS", [],
1399
+ ["container",
1400
+ "code_interpreter",
1401
+ "web_search_preview"]),
1402
}
1403
1404
# --8<-- [end:env-vars-definition]