A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[gpt-oss] add demo tool server (#22393)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
Chen Zhang committed
Aug 6, 2025 at 17:47 UTC
19c9365aa48d514ae6ef45242359dc98c6046666
4 files changed
+85
vllm/entrypoints/openai/api_server.py
+7
@@ -92,6 +92,7 @@ from vllm.entrypoints.openai.serving_tokenization import (
92
from vllm.entrypoints.openai.serving_transcription import (
93
OpenAIServingTranscription, OpenAIServingTranslation)
94
from vllm.entrypoints.openai.tool_parsers import ToolParserManager
95
+from vllm.entrypoints.tool_server import DemoToolServer, ToolServer
96
from vllm.entrypoints.utils import (cli_env_setup, load_aware_call,
97
log_non_default_args, with_cancellation)
98
from vllm.logger import init_logger
@@ -1620,6 +1621,11 @@ async def init_app_state(
1621
"This discrepancy may lead to performance degradation.",
1622
resolved_chat_template, args.model)
1623
1624
+ if args.tool_server == "demo":
1625
+ tool_server: Optional[ToolServer] = DemoToolServer()
1626
+ else:
1627
+ tool_server = None
1628
+
1629
# Merge default_mm_loras into the static lora_modules
1630
default_mm_loras = (vllm_config.lora_config.default_mm_loras
1631
if vllm_config.lora_config is not None else {})
@@ -1654,6 +1660,7 @@ async def init_app_state(
1660
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
1661
enable_auto_tools=args.enable_auto_tool_choice,
1662
tool_parser=args.tool_call_parser,
1663
+ tool_server=tool_server,
1664
reasoning_parser=args.reasoning_parser,
1665
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
1666
enable_force_include_usage=args.enable_force_include_usage,
vllm/entrypoints/openai/cli_args.py
+4
@@ -147,6 +147,10 @@ schema. Example: `[{"type": "text", "text": "Hello world!"}]`"""
147
"""Special the tool parser plugin write to parse the model-generated tool
148
into OpenAI API format, the name register in this plugin can be used in
149
`--tool-call-parser`."""
150
+ tool_server: Optional[str] = None
151
+ """Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
152
+ Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for demo
153
+ purpose."""
154
log_config_file: Optional[str] = envs.VLLM_LOGGING_CONFIG_PATH
155
"""Path to logging config JSON file for both vllm and uvicorn"""
156
max_log_len: Optional[int] = None
vllm/entrypoints/openai/serving_responses.py
+4
@@ -29,6 +29,7 @@ from vllm.entrypoints.openai.protocol import (ErrorResponse,
29
# yapf: enable
30
from vllm.entrypoints.openai.serving_engine import OpenAIServing
31
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
32
+from vllm.entrypoints.tool_server import ToolServer
33
from vllm.logger import init_logger
34
from vllm.reasoning import ReasoningParser, ReasoningParserManager
35
from vllm.sampling_params import SamplingParams
@@ -53,6 +54,7 @@ class OpenAIServingResponses(OpenAIServing):
54
reasoning_parser: str = "",
55
enable_auto_tools: bool = False,
56
tool_parser: Optional[str] = None,
57
+ tool_server: Optional[ToolServer] = None,
58
enable_prompt_tokens_details: bool = False,
59
enable_force_include_usage: bool = False,
60
) -> None:
@@ -114,6 +116,8 @@ class OpenAIServingResponses(OpenAIServing):
116
117
self.background_tasks: dict[str, asyncio.Task] = {}
118
119
+ self.tool_server = tool_server
120
+
121
async def create_responses(
122
self,
123
request: ResponsesRequest,
vllm/entrypoints/tool_server.py
new
+70
@@ -0,0 +1,70 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from abc import ABC, abstractmethod
4
+from contextlib import AbstractAsyncContextManager, asynccontextmanager
5
+from typing import Any, Optional
6
+
7
+from openai_harmony import ToolNamespaceConfig
8
+
9
+from vllm.entrypoints.tool import HarmonyBrowserTool, HarmonyPythonTool, Tool
10
+from vllm.logger import init_logger
11
+
12
+logger = init_logger(__name__)
13
+
14
+
15
+class ToolServer(ABC):
16
+
17
+ @abstractmethod
18
+ def has_tool(self, tool_name: str) -> bool:
19
+ """
20
+ Return True if the tool is supported, False otherwise.
21
+ """
22
+ pass
23
+
24
+ @abstractmethod
25
+ def get_tool_description(self,
26
+ tool_name: str) -> Optional[ToolNamespaceConfig]:
27
+ """
28
+ Return the tool description for the given tool name.
29
+ If the tool is not supported, return None.
30
+ """
31
+ pass
32
+
33
+ @abstractmethod
34
+ def new_session(self, tool_name: str) -> AbstractAsyncContextManager[Any]:
35
+ """
36
+ Create a session for the tool.
37
+ """
38
+ ...
39
+
40
+
41
+class DemoToolServer(ToolServer):
42
+
43
+ def __init__(self):
44
+ self.tools: dict[str, Tool] = {}
45
+ browser_tool = HarmonyBrowserTool()
46
+ if browser_tool.enabled:
47
+ self.tools["browser"] = browser_tool
48
+ python_tool = HarmonyPythonTool()
49
+ if python_tool.enabled:
50
+ self.tools["python"] = python_tool
51
+ logger.info("DemoToolServer initialized with tools: %s",
52
+ list(self.tools.keys()))
53
+
54
+ def has_tool(self, tool_name: str) -> bool:
55
+ return tool_name in self.tools
56
+
57
+ def get_tool_description(self,
58
+ tool_name: str) -> Optional[ToolNamespaceConfig]:
59
+ if tool_name not in self.tools:
60
+ return None
61
+ if tool_name == "browser":
62
+ return ToolNamespaceConfig.browser()
63
+ elif tool_name == "python":
64
+ return ToolNamespaceConfig.python()
65
+ else:
66
+ raise ValueError(f"Unknown tool {tool_name}")
67
+
68
+ @asynccontextmanager
69
+ async def new_session(self, tool_name: str):
70
+ yield self.tools[tool_name]