A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 18:30 UTC
[Mypy] Fix adjust_request typing (#38264)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
Mar 31, 2026 at 00:21 UTC
3e802e8786e05ba94e8a585630f14c86f83b8ad4
14 files changed
+49
-17
docs/features/tool_calling.md
+1
-1
@@ -505,7 +505,7 @@ Here is a summary of a plugin file:
505
506
# adjust request. e.g.: set skip special tokens
507
# to False for tool call output.
508
- def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
508
+ def adjust_request(self, request: ChatCompletionRequest | ResponsesRequest) -> ChatCompletionRequest | ResponsesRequest:
509
return request
510
511
# implement the tool call parse for stream call
vllm/entrypoints/serve/render/serving.py
+1
-1
@@ -546,7 +546,7 @@ class OpenAIServingRender:
546
raise NotImplementedError(msg)
547
tokenizer = renderer.get_tokenizer()
548
request = tool_parser(tokenizer, request.tools).adjust_request(
549
- request=request # type: ignore[arg-type]
549
+ request=request
550
)
551
552
return conversation, [engine_input]
vllm/parser/abstract_parser.py
+4
-4
@@ -32,9 +32,7 @@ from vllm.entrypoints.openai.engine.protocol import (
32
FunctionCall,
33
FunctionDefinition,
34
)
35
-from vllm.entrypoints.openai.responses.protocol import (
36
- ResponsesRequest,
37
-)
35
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
36
from vllm.logger import init_logger
37
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
38
from vllm.tokenizers import TokenizerLike
@@ -229,7 +227,9 @@ class Parser:
227
228
# ========== Tool Parser Methods ==========
229
232
- def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
230
+ def adjust_request(
231
+ self, request: ChatCompletionRequest | ResponsesRequest
232
+ ) -> ChatCompletionRequest | ResponsesRequest:
233
"""
234
Adjust the request parameters for tool calling.
235
vllm/tool_parsers/abstract_tool_parser.py
+3
-1
@@ -62,7 +62,9 @@ class ToolParser:
62
# whereas all tokenizers have .get_vocab()
63
return self.model_tokenizer.get_vocab()
64
65
- def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
65
+ def adjust_request(
66
+ self, request: ChatCompletionRequest | ResponsesRequest
67
+ ) -> ChatCompletionRequest | ResponsesRequest:
68
"""
69
Static method that used to adjust the request parameters.
70
"""
vllm/tool_parsers/deepseekv32_tool_parser.py
+4
-1
@@ -19,6 +19,7 @@ from vllm.entrypoints.openai.engine.protocol import (
19
FunctionCall,
20
ToolCall,
21
)
22
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
23
from vllm.logger import init_logger
24
from vllm.tokenizers import TokenizerLike
25
from vllm.tool_parsers.abstract_tool_parser import (
@@ -78,7 +79,9 @@ class DeepSeekV32ToolParser(ToolParser):
79
"vLLM Successfully import tool parser %s !", self.__class__.__name__
80
)
81
81
- def adjust_request(self, request):
82
+ def adjust_request(
83
+ self, request: ChatCompletionRequest | ResponsesRequest
84
+ ) -> ChatCompletionRequest | ResponsesRequest:
85
request = super().adjust_request(request)
86
if request.tools and request.tool_choice != "none":
87
# Ensure tool call tokens
vllm/tool_parsers/functiongemma_tool_parser.py
+4
-1
@@ -18,6 +18,7 @@ from vllm.entrypoints.openai.engine.protocol import (
18
FunctionCall,
19
ToolCall,
20
)
21
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
22
from vllm.logger import init_logger
23
from vllm.tokenizers import TokenizerLike
24
from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser
@@ -86,7 +87,9 @@ class FunctionGemmaToolParser(ToolParser):
87
88
return arguments
89
89
- def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
90
+ def adjust_request(
91
+ self, request: ChatCompletionRequest | ResponsesRequest
92
+ ) -> ChatCompletionRequest | ResponsesRequest:
93
request = super().adjust_request(request)
94
if request.tools and request.tool_choice != "none":
95
request.skip_special_tokens = False
vllm/tool_parsers/gigachat3_tool_parser.py
+4
-1
@@ -18,6 +18,7 @@ from vllm.entrypoints.openai.engine.protocol import (
18
FunctionCall,
19
ToolCall,
20
)
21
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
22
from vllm.logger import init_logger
23
from vllm.tokenizers import TokenizerLike
24
from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser
@@ -55,7 +56,9 @@ class GigaChat3ToolParser(ToolParser):
56
self.end_content: bool = False
57
self.streamed_args_for_tool: list[str] = []
58
58
- def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
59
+ def adjust_request(
60
+ self, request: ChatCompletionRequest | ResponsesRequest
61
+ ) -> ChatCompletionRequest | ResponsesRequest:
62
request = super().adjust_request(request)
63
if request.tools and request.tool_choice != "none":
64
request.skip_special_tokens = False
vllm/tool_parsers/glm4_moe_tool_parser.py
+4
-1
@@ -30,6 +30,7 @@ from vllm.entrypoints.openai.engine.protocol import (
30
FunctionCall,
31
ToolCall,
32
)
33
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
34
from vllm.logger import init_logger
35
from vllm.tokenizers import TokenizerLike
36
from vllm.tool_parsers.abstract_tool_parser import (
@@ -151,7 +152,9 @@ class Glm4MoeModelToolParser(ToolParser):
152
logger.exception("Failed to determine if tools are enabled.")
153
return False
154
154
- def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
155
+ def adjust_request(
156
+ self, request: ChatCompletionRequest | ResponsesRequest
157
+ ) -> ChatCompletionRequest | ResponsesRequest:
158
"""Adjust request parameters for tool call token handling."""
159
request = super().adjust_request(request)
160
if request.tools and request.tool_choice != "none":
vllm/tool_parsers/granite4_tool_parser.py
+4
-1
@@ -19,6 +19,7 @@ from vllm.entrypoints.openai.engine.protocol import (
19
FunctionCall,
20
ToolCall,
21
)
22
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
23
from vllm.logger import init_logger
24
from vllm.tokenizers import TokenizerLike
25
from vllm.tool_parsers.abstract_tool_parser import (
@@ -59,7 +60,9 @@ class Granite4ToolParser(ToolParser):
60
self.start_regex = re.compile(self.tc_start)
61
self.end_regex = re.compile(self.tc_end)
62
62
- def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
63
+ def adjust_request(
64
+ self, request: ChatCompletionRequest | ResponsesRequest
65
+ ) -> ChatCompletionRequest | ResponsesRequest:
66
request = super().adjust_request(request)
67
if request.tools and request.tool_choice != "none":
68
# do not skip special tokens because the tool_call tokens are
vllm/tool_parsers/hermes_tool_parser.py
+4
-1
@@ -18,6 +18,7 @@ from vllm.entrypoints.openai.engine.protocol import (
18
FunctionCall,
19
ToolCall,
20
)
21
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
22
from vllm.logger import init_logger
23
from vllm.tokenizers import TokenizerLike
24
from vllm.tool_parsers.abstract_tool_parser import (
@@ -77,7 +78,9 @@ class Hermes2ProToolParser(ToolParser):
78
# Streaming state: what has been sent to the client.
79
self._sent_content_idx: int = 0
80
80
- def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
81
+ def adjust_request(
82
+ self, request: ChatCompletionRequest | ResponsesRequest
83
+ ) -> ChatCompletionRequest | ResponsesRequest:
84
request = super().adjust_request(request)
85
if request.tools and request.tool_choice != "none":
86
# do not skip special tokens because the tool_call tokens are
vllm/tool_parsers/internlm2_tool_parser.py
+4
-1
@@ -19,6 +19,7 @@ from vllm.entrypoints.openai.engine.protocol import (
19
FunctionCall,
20
ToolCall,
21
)
22
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
23
from vllm.logger import init_logger
24
from vllm.tokenizers import TokenizerLike
25
from vllm.tool_parsers.abstract_tool_parser import (
@@ -35,7 +36,9 @@ class Internlm2ToolParser(ToolParser):
36
super().__init__(tokenizer, tools)
37
self.position = 0
38
38
- def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
39
+ def adjust_request(
40
+ self, request: ChatCompletionRequest | ResponsesRequest
41
+ ) -> ChatCompletionRequest | ResponsesRequest:
42
request = super().adjust_request(request)
43
if request.tools and request.tool_choice != "none":
44
# do not skip special tokens because internlm use the special
vllm/tool_parsers/jamba_tool_parser.py
+4
-1
@@ -20,6 +20,7 @@ from vllm.entrypoints.openai.engine.protocol import (
20
FunctionCall,
21
ToolCall,
22
)
23
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
24
from vllm.logger import init_logger
25
from vllm.tokenizers import TokenizerLike
26
from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser
@@ -68,7 +69,9 @@ class JambaToolParser(ToolParser):
69
"tokens in the tokenizer!"
70
)
71
71
- def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
72
+ def adjust_request(
73
+ self, request: ChatCompletionRequest | ResponsesRequest
74
+ ) -> ChatCompletionRequest | ResponsesRequest:
75
request = super().adjust_request(request)
76
if request.tools and request.tool_choice != "none":
77
# do not skip special tokens because jamba use the special
vllm/tool_parsers/mistral_tool_parser.py
+4
-1
@@ -23,6 +23,7 @@ from vllm.entrypoints.openai.engine.protocol import (
23
FunctionCall,
24
ToolCall,
25
)
26
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
27
from vllm.logger import init_logger
28
from vllm.tokenizers import TokenizerLike
29
from vllm.tool_parsers.abstract_tool_parser import (
@@ -111,7 +112,9 @@ class MistralToolParser(ToolParser):
112
"the tokenizer!"
113
)
114
114
- def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
115
+ def adjust_request(
116
+ self, request: ChatCompletionRequest | ResponsesRequest
117
+ ) -> ChatCompletionRequest | ResponsesRequest:
118
request = super().adjust_request(request)
119
if (
120
not is_mistral_tokenizer(self.model_tokenizer)
vllm/tool_parsers/step3_tool_parser.py
+4
-1
@@ -19,6 +19,7 @@ from vllm.entrypoints.openai.engine.protocol import (
19
FunctionCall,
20
ToolCall,
21
)
22
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
23
from vllm.logger import init_logger
24
from vllm.tokenizers import TokenizerLike
25
from vllm.tool_parsers.abstract_tool_parser import (
@@ -51,7 +52,9 @@ class Step3ToolParser(ToolParser):
52
self.tool_block_started = False
53
self.tool_block_finished = False
54
54
- def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
55
+ def adjust_request(
56
+ self, request: ChatCompletionRequest | ResponsesRequest
57
+ ) -> ChatCompletionRequest | ResponsesRequest:
58
request = super().adjust_request(request)
59
if request.tools and request.tool_choice != "none":
60
request.skip_special_tokens = False