A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Refactor]Reduce duplicate code in serving_chat (#26627)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Oct 11, 2025 at 20:04 UTC
d0bed837ac83aa33f6540d32cbcc75525328078e
3 files changed
+56
-49
vllm/entrypoints/openai/serving_chat.py
+8
-36
@@ -6,7 +6,7 @@ import json
6
import time
7
from collections.abc import AsyncGenerator, AsyncIterator
8
from collections.abc import Sequence as GenericSequence
9
-from typing import Callable, Final, Optional, Union
9
+from typing import Final, Optional, Union
10
11
import jinja2
12
import partial_json_parser
@@ -56,14 +56,13 @@ from vllm.entrypoints.openai.protocol import (
56
)
57
from vllm.entrypoints.openai.serving_engine import OpenAIServing, clamp_prompt_logprobs
58
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
59
-from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
59
+from vllm.entrypoints.openai.tool_parsers import ToolParser
60
from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import MistralToolCall
61
from vllm.entrypoints.utils import get_max_tokens
62
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
63
from vllm.logger import init_logger
64
from vllm.logprobs import Logprob
65
from vllm.outputs import CompletionOutput, RequestOutput
66
-from vllm.reasoning import ReasoningParser, ReasoningParserManager
66
from vllm.sampling_params import BeamSearchParams, SamplingParams
67
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
68
from vllm.transformers_utils.tokenizers import (
@@ -112,42 +111,15 @@ class OpenAIServingChat(OpenAIServing):
111
self.trust_request_chat_template = trust_request_chat_template
112
self.enable_log_outputs = enable_log_outputs
113
114
+ # set up reasoning parser
115
+ self.reasoning_parser = self._get_reasoning_parser(
116
+ reasoning_parser_name=reasoning_parser
117
+ )
118
# set up tool use
119
self.enable_auto_tools: bool = enable_auto_tools
117
- if self.enable_auto_tools:
118
- logger.info(
119
- '"auto" tool choice has been enabled please note that while'
120
- " the parallel_tool_calls client option is preset for "
121
- "compatibility reasons, it will be ignored."
122
- )
123
-
124
- self.reasoning_parser: Optional[Callable[[AnyTokenizer], ReasoningParser]] = (
125
- None
120
+ self.tool_parser = self._get_tool_parser(
121
+ tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
122
)
127
- if reasoning_parser:
128
- try:
129
- self.reasoning_parser = ReasoningParserManager.get_reasoning_parser(
130
- reasoning_parser
131
- )
132
- assert self.reasoning_parser is not None
133
- except Exception as e:
134
- raise TypeError(f"{reasoning_parser=} has not been registered") from e
135
- self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
136
- if self.enable_auto_tools:
137
- try:
138
- if tool_parser == "pythonic" and self.model_config.model.startswith(
139
- "meta-llama/Llama-3.2"
140
- ):
141
- logger.warning(
142
- "Llama3.2 models may struggle to emit valid pythonic tool calls"
143
- )
144
- self.tool_parser = ToolParserManager.get_tool_parser(tool_parser)
145
- except Exception as e:
146
- raise TypeError(
147
- "Error: --enable-auto-tool-choice requires "
148
- f"tool_parser:'{tool_parser}' which has not "
149
- "been registered"
150
- ) from e
123
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
124
125
self.enable_prompt_tokens_details = enable_prompt_tokens_details
vllm/entrypoints/openai/serving_engine.py
+46
-1
@@ -63,7 +63,7 @@ from vllm.entrypoints.openai.protocol import (
63
TranslationRequest,
64
)
65
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
66
-from vllm.entrypoints.openai.tool_parsers import ToolParser
66
+from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
67
from vllm.entrypoints.renderer import BaseRenderer, CompletionRenderer, RenderConfig
68
from vllm.entrypoints.utils import _validate_truncation_size
69
from vllm.inputs.data import PromptType
@@ -82,6 +82,7 @@ from vllm.multimodal import ( # noqa: F401 - Required to resolve Pydantic error
82
)
83
from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
84
from vllm.pooling_params import PoolingParams
85
+from vllm.reasoning import ReasoningParser, ReasoningParserManager
86
from vllm.sampling_params import BeamSearchParams, SamplingParams
87
from vllm.tracing import (
88
contains_trace_headers,
@@ -274,6 +275,50 @@ class OpenAIServing:
275
self.model_config = self.models.model_config
276
self.max_model_len = self.model_config.max_model_len
277
278
+ def _get_tool_parser(
279
+ self, tool_parser_name: Optional[str] = None, enable_auto_tools: bool = False
280
+ ) -> Optional[Callable[[AnyTokenizer], ToolParser]]:
281
+ """Get the tool parser based on the name."""
282
+ parser = None
283
+ if not enable_auto_tools or tool_parser_name is None:
284
+ return parser
285
+ logger.info(
286
+ '"auto" tool choice has been enabled please note that while'
287
+ " the parallel_tool_calls client option is preset for "
288
+ "compatibility reasons, it will be ignored."
289
+ )
290
+
291
+ try:
292
+ if tool_parser_name == "pythonic" and self.model_config.model.startswith(
293
+ "meta-llama/Llama-3.2"
294
+ ):
295
+ logger.warning(
296
+ "Llama3.2 models may struggle to emit valid pythonic tool calls"
297
+ )
298
+ parser = ToolParserManager.get_tool_parser(tool_parser_name)
299
+ except Exception as e:
300
+ raise TypeError(
301
+ "Error: --enable-auto-tool-choice requires "
302
+ f"tool_parser:'{tool_parser_name}' which has not "
303
+ "been registered"
304
+ ) from e
305
+ return parser
306
+
307
+ def _get_reasoning_parser(
308
+ self,
309
+ reasoning_parser_name: str,
310
+ ) -> Optional[Callable[[AnyTokenizer], ReasoningParser]]:
311
+ """Get the reasoning parser based on the name."""
312
+ parser = None
313
+ if not reasoning_parser_name:
314
+ return None
315
+ try:
316
+ parser = ReasoningParserManager.get_reasoning_parser(reasoning_parser_name)
317
+ assert parser is not None
318
+ except Exception as e:
319
+ raise TypeError(f"{reasoning_parser_name=} has not been registered") from e
320
+ return parser
321
+
322
async def reset_mm_cache(self) -> None:
323
self.processor.clear_mm_cache()
324
await self.engine_client.reset_mm_cache()
vllm/entrypoints/openai/serving_responses.py
+2
-12
@@ -96,7 +96,6 @@ from vllm.logger import init_logger
96
from vllm.logprobs import Logprob as SampleLogprob
97
from vllm.logprobs import SampleLogprobs
98
from vllm.outputs import CompletionOutput
99
-from vllm.reasoning import ReasoningParser, ReasoningParserManager
99
from vllm.sampling_params import SamplingParams
100
from vllm.transformers_utils.tokenizer import AnyTokenizer
101
from vllm.utils import random_uuid
@@ -136,18 +135,9 @@ class OpenAIServingResponses(OpenAIServing):
135
self.chat_template_content_format: Final = chat_template_content_format
136
self.enable_log_outputs = enable_log_outputs
137
139
- self.reasoning_parser: Optional[Callable[[AnyTokenizer], ReasoningParser]] = (
140
- None
138
+ self.reasoning_parser = self._get_reasoning_parser(
139
+ reasoning_parser_name=reasoning_parser
140
)
142
- if reasoning_parser:
143
- try:
144
- self.reasoning_parser = ReasoningParserManager.get_reasoning_parser(
145
- reasoning_parser
146
- )
147
- assert self.reasoning_parser is not None
148
- except Exception as e:
149
- raise TypeError(f"{reasoning_parser=} has not been registered") from e
150
-
141
self.enable_prompt_tokens_details = enable_prompt_tokens_details
142
self.enable_force_include_usage = enable_force_include_usage
143
self.default_sampling_params = self.model_config.get_diff_sampling_param()