@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e1bf04b6c2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 8, 22:45 UTC

[1/N] Initial Implementation of Parser for ResponsesAPI (#32712)

Signed-off-by: Andrew Xia <axia@fb.com> Co-authored-by: Andrew Xia <axia@fb.com>

Andrew Xia committed Feb 3, 2026 at 21:59 UTC e1bf04b6c27a070859264290ffdccbf333f27fa6
10 files changed +772 -61
tests/entrypoints/openai/test_chat_error.py
+1
@@ -36,6 +36,7 @@ class MockHFConfig:
36 class MockModelConfig:
37 task = "generate"
38 runner_type = "generate"
39 + model = MODEL_NAME
40 tokenizer = MODEL_NAME
41 trust_remote_code = False
42 tokenizer_mode = "auto"
tests/entrypoints/openai/test_completion_error.py
+1
@@ -36,6 +36,7 @@ class MockHFConfig:
36 class MockModelConfig:
37 task = "generate"
38 runner_type = "generate"
39 + model = MODEL_NAME
40 tokenizer = MODEL_NAME
41 trust_remote_code = False
42 tokenizer_mode = "auto"
tests/entrypoints/openai/test_serving_chat.py
+1
@@ -511,6 +511,7 @@ class MockHFConfig:
511 class MockModelConfig:
512 task = "generate"
513 runner_type = "generate"
514 + model = MODEL_NAME
515 tokenizer = MODEL_NAME
516 trust_remote_code = False
517 tokenizer_mode = "auto"
vllm/entrypoints/openai/chat_completion/serving.py
+6 -3
@@ -71,6 +71,7 @@ from vllm.inputs.data import EmbedsPrompt, TokensPrompt
71 from vllm.logger import init_logger
72 from vllm.logprobs import Logprob
73 from vllm.outputs import CompletionOutput, RequestOutput
74 +from vllm.parser import ParserManager
75 from vllm.sampling_params import BeamSearchParams, SamplingParams
76 from vllm.tokenizers import TokenizerLike
77 from vllm.tokenizers.mistral import (
@@ -131,13 +132,15 @@ class OpenAIServingChat(OpenAIServing):
132 self.logits_processors = self.model_config.logits_processors
133
134 # set up reasoning parser
134 - self.reasoning_parser = self._get_reasoning_parser(
135 + self.reasoning_parser = ParserManager.get_reasoning_parser(
136 reasoning_parser_name=reasoning_parser
137 )
138 # set up tool use
139 self.enable_auto_tools: bool = enable_auto_tools
139 - self.tool_parser = self._get_tool_parser(
140 - tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
140 + self.tool_parser = ParserManager.get_tool_parser(
141 + tool_parser_name=tool_parser,
142 + enable_auto_tools=enable_auto_tools,
143 + model_name=self.model_config.model,
144 )
145 self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
146
vllm/entrypoints/openai/engine/serving.py
+1 -42
@@ -107,11 +107,10 @@ from vllm.lora.request import LoRARequest
107 from vllm.multimodal import MultiModalDataDict
108 from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
109 from vllm.pooling_params import PoolingParams
110 -from vllm.reasoning import ReasoningParser, ReasoningParserManager
110 from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
111 from vllm.sampling_params import BeamSearchParams, SamplingParams
112 from vllm.tokenizers import TokenizerLike
114 -from vllm.tool_parsers import ToolParser, ToolParserManager
113 +from vllm.tool_parsers import ToolParser
114 from vllm.tracing import (
115 contains_trace_headers,
116 extract_trace_headers,
@@ -246,46 +245,6 @@ class OpenAIServing:
245 self.model_config = self.models.model_config
246 self.max_model_len = self.model_config.max_model_len
247
249 - def _get_tool_parser(
250 - self, tool_parser_name: str | None = None, enable_auto_tools: bool = False
251 - ) -> Callable[[TokenizerLike], ToolParser] | None:
252 - """Get the tool parser based on the name."""
253 - parser = None
254 - if not enable_auto_tools or tool_parser_name is None:
255 - return parser
256 - logger.info('"auto" tool choice has been enabled.')
257 -
258 - try:
259 - if tool_parser_name == "pythonic" and self.model_config.model.startswith(
260 - "meta-llama/Llama-3.2"
261 - ):
262 - logger.warning(
263 - "Llama3.2 models may struggle to emit valid pythonic tool calls"
264 - )
265 - parser = ToolParserManager.get_tool_parser(tool_parser_name)
266 - except Exception as e:
267 - raise TypeError(
268 - "Error: --enable-auto-tool-choice requires "
269 - f"tool_parser:'{tool_parser_name}' which has not "
270 - "been registered"
271 - ) from e
272 - return parser
273 -
274 - def _get_reasoning_parser(
275 - self,
276 - reasoning_parser_name: str,
277 - ) -> Callable[[TokenizerLike], ReasoningParser] | None:
278 - """Get the reasoning parser based on the name."""
279 - parser = None
280 - if not reasoning_parser_name:
281 - return None
282 - try:
283 - parser = ReasoningParserManager.get_reasoning_parser(reasoning_parser_name)
284 - assert parser is not None
285 - except Exception as e:
286 - raise TypeError(f"{reasoning_parser_name=} has not been registered") from e
287 - return parser
288 -
248 async def beam_search(
249 self,
250 prompt: PromptType,
vllm/entrypoints/openai/responses/serving.py
+22 -16
@@ -123,6 +123,7 @@ from vllm.logger import init_logger
123 from vllm.logprobs import Logprob as SampleLogprob
124 from vllm.logprobs import SampleLogprobs
125 from vllm.outputs import CompletionOutput
126 +from vllm.parser import ParserManager
127 from vllm.sampling_params import SamplingParams, StructuredOutputsParams
128 from vllm.tokenizers import TokenizerLike
129 from vllm.utils import random_uuid
@@ -217,8 +218,13 @@ class OpenAIServingResponses(OpenAIServing):
218 self.chat_template_content_format: Final = chat_template_content_format
219 self.enable_log_outputs = enable_log_outputs
220
220 - self.reasoning_parser = self._get_reasoning_parser(
221 - reasoning_parser_name=reasoning_parser
221 + # Set up the unified parser - either a unified parser or fall back to
222 + # separate parsers accessed through the parser interface
223 + self.parser = ParserManager.get_parser(
224 + tool_parser_name=tool_parser,
225 + reasoning_parser_name=reasoning_parser,
226 + enable_auto_tools=enable_auto_tools,
227 + model_name=self.model_config.model,
228 )
229 self.enable_prompt_tokens_details = enable_prompt_tokens_details
230 self.enable_force_include_usage = enable_force_include_usage
@@ -263,10 +269,6 @@ class OpenAIServingResponses(OpenAIServing):
269 self.tool_call_id_type = "random"
270
271 self.enable_auto_tools = enable_auto_tools
266 - # set up tool use
267 - self.tool_parser = self._get_tool_parser(
268 - tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
269 - )
272 # HACK(woosuk): This is a hack. We should use a better store.
273 # FIXME: If enable_store=True, this may cause a memory leak since we
274 # never remove responses from the store.
@@ -469,9 +471,13 @@ class OpenAIServingResponses(OpenAIServing):
471 context = ParsableContext(
472 response_messages=messages,
473 tokenizer=tokenizer,
472 - reasoning_parser_cls=self.reasoning_parser,
474 + reasoning_parser_cls=self.parser.reasoning_parser_cls
475 + if self.parser
476 + else None,
477 request=request,
474 - tool_parser_cls=self.tool_parser,
478 + tool_parser_cls=self.parser.tool_parser_cls
479 + if self.parser
480 + else None,
481 available_tools=available_tools,
482 chat_template=self.chat_template,
483 chat_template_content_format=self.chat_template_content_format,
@@ -479,8 +485,8 @@ class OpenAIServingResponses(OpenAIServing):
485 else:
486 context = SimpleContext()
487
482 - if self.reasoning_parser is not None:
483 - reasoning_parser = self.reasoning_parser(tokenizer)
488 + if self.parser and self.parser.reasoning_parser_cls is not None:
489 + reasoning_parser = self.parser.reasoning_parser_cls(tokenizer)
490 if (
491 isinstance(
492 struct_out := sampling_params.structured_outputs,
@@ -617,7 +623,7 @@ class OpenAIServingResponses(OpenAIServing):
623 default_template_content_format=self.chat_template_content_format,
624 default_template_kwargs=None,
625 tool_dicts=tool_dicts,
620 - tool_parser=self.tool_parser,
626 + tool_parser=self.parser.tool_parser_cls if self.parser else None,
627 )
628 return messages, engine_prompts
629
@@ -909,9 +915,9 @@ class OpenAIServingResponses(OpenAIServing):
915 final_output: CompletionOutput,
916 tokenizer: TokenizerLike,
917 ) -> list[ResponseOutputItem]:
912 - if self.reasoning_parser:
918 + if self.parser and self.parser.reasoning_parser_cls:
919 try:
914 - reasoning_parser = self.reasoning_parser(tokenizer)
920 + reasoning_parser = self.parser.reasoning_parser_cls(tokenizer)
921 except RuntimeError as e:
922 logger.exception("Error in reasoning parser creation.")
923 raise e
@@ -958,7 +964,7 @@ class OpenAIServingResponses(OpenAIServing):
964 tokenizer=tokenizer,
965 content=content,
966 enable_auto_tools=self.enable_auto_tools,
961 - tool_parser_cls=self.tool_parser,
967 + tool_parser_cls=self.parser.tool_parser_cls if self.parser else None,
968 )
969
970 if content or (self.use_harmony and tool_calls):
@@ -1339,8 +1345,8 @@ class OpenAIServingResponses(OpenAIServing):
1345 current_output_index = 0
1346 current_item_id = ""
1347 reasoning_parser = None
1342 - if self.reasoning_parser:
1343 - reasoning_parser = self.reasoning_parser(tokenizer)
1348 + if self.parser and self.parser.reasoning_parser_cls:
1349 + reasoning_parser = self.parser.reasoning_parser_cls(tokenizer)
1350 previous_text = ""
1351 previous_token_ids: list[int] = []
1352 first_delta_sent = False
vllm/parser/__init__.py new
+39
@@ -0,0 +1,39 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from vllm.parser.abstract_parser import (
5 + DelegatingParser,
6 + Parser,
7 + _WrappedParser,
8 +)
9 +from vllm.parser.parser_manager import ParserManager
10 +
11 +__all__ = [
12 + "Parser",
13 + "DelegatingParser",
14 + "ParserManager",
15 + "_WrappedParser",
16 +]
17 +
18 +_PARSERS_TO_REGISTER = {
19 + "minimax_m2": ( # name
20 + "minimax_m2_parser", # filename
21 + "MiniMaxM2Parser", # class_name
22 + ),
23 +}
24 +
25 +# Register lazy parsers
26 +ParserManager.register_lazy_module(
27 + name="minimax_m2",
28 + module_path="vllm.parser.minimax_m2_parser",
29 + class_name="MiniMaxM2Parser",
30 +)
31 +
32 +
33 +def register_lazy_parsers():
34 + for name, (file_name, class_name) in _PARSERS_TO_REGISTER.items():
35 + module_path = f"vllm.parser.{file_name}"
36 + ParserManager.register_lazy_module(name, module_path, class_name)
37 +
38 +
39 +register_lazy_parsers()
vllm/parser/abstract_parser.py new
+341
@@ -0,0 +1,341 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from abc import abstractmethod
5 +from collections.abc import Sequence
6 +from functools import cached_property
7 +
8 +from vllm.entrypoints.openai.chat_completion.protocol import (
9 + ChatCompletionRequest,
10 +)
11 +from vllm.entrypoints.openai.engine.protocol import (
12 + DeltaMessage,
13 + ExtractedToolCallInformation,
14 +)
15 +from vllm.entrypoints.openai.responses.protocol import (
16 + ResponsesRequest,
17 +)
18 +from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
19 +from vllm.tokenizers import TokenizerLike
20 +from vllm.tool_parsers.abstract_tool_parser import ToolParser
21 +
22 +
23 +class Parser:
24 + """
25 + Abstract Parser class that unifies ReasoningParser and ToolParser into
26 + a single interface for parsing model output.
27 +
28 + This class provides a unified way to handle both reasoning extraction
29 + (e.g., chain-of-thought content in <think> tags) and tool call extraction
30 + (e.g., function calls in XML/JSON format) from model outputs.
31 +
32 + Subclasses can either:
33 + 1. Override the abstract methods directly for custom parsing logic
34 + 2. Set `reasoning_parser` and `tool_parser` properties to delegate to
35 + existing parser implementations
36 +
37 + Class Attributes:
38 + reasoning_parser_cls: The ReasoningParser class to use (for compatibility
39 + with code that needs the class, not instance).
40 + tool_parser_cls: The ToolParser class to use (for compatibility with
41 + code that needs the class, not instance).
42 + """
43 +
44 + # Class-level parser classes for compatibility with existing patterns
45 + # Subclasses should override these if they use specific parser classes
46 + reasoning_parser_cls: type[ReasoningParser] | None = None
47 + tool_parser_cls: type[ToolParser] | None = None
48 +
49 + def __init__(self, tokenizer: TokenizerLike, *args, **kwargs):
50 + """
51 + Initialize the Parser.
52 +
53 + Args:
54 + tokenizer: The tokenizer used by the model. This is required for
55 + token-based parsing operations.
56 + """
57 + self.model_tokenizer = tokenizer
58 + self._reasoning_parser: ReasoningParser | None = None
59 + self._tool_parser: ToolParser | None = None
60 +
61 + @cached_property
62 + def vocab(self) -> dict[str, int]:
63 + """Get the vocabulary mapping from tokens to IDs."""
64 + return self.model_tokenizer.get_vocab()
65 +
66 + @property
67 + def reasoning_parser(self) -> ReasoningParser | None:
68 + """The underlying reasoning parser, if any."""
69 + return self._reasoning_parser
70 +
71 + @reasoning_parser.setter
72 + def reasoning_parser(self, parser: ReasoningParser | None) -> None:
73 + self._reasoning_parser = parser
74 +
75 + @property
76 + def tool_parser(self) -> ToolParser | None:
77 + """The underlying tool parser, if any."""
78 + return self._tool_parser
79 +
80 + @tool_parser.setter
81 + def tool_parser(self, parser: ToolParser | None) -> None:
82 + self._tool_parser = parser
83 +
84 + # ========== Reasoning Parser Methods ==========
85 +
86 + @abstractmethod
87 + def is_reasoning_end(self, input_ids: list[int]) -> bool:
88 + """
89 + Check if the reasoning content ends in the input_ids.
90 +
91 + Used by structured engines like `xgrammar` to check if the
92 + reasoning content ends in the model output.
93 +
94 + Args:
95 + input_ids: The token IDs of the model output.
96 +
97 + Returns:
98 + True if the reasoning content ends in the input_ids.
99 + """
100 +
101 + def is_reasoning_end_streaming(
102 + self, input_ids: list[int], delta_ids: list[int]
103 + ) -> bool:
104 + """
105 + Check if the reasoning content ends during a decode step.
106 +
107 + Args:
108 + input_ids: The entire model output token IDs.
109 + delta_ids: The last few computed tokens at the current decode step.
110 +
111 + Returns:
112 + True if the reasoning content ends in the delta_ids.
113 + """
114 + return self.is_reasoning_end(input_ids)
115 +
116 + @abstractmethod
117 + def extract_content_ids(self, input_ids: list[int]) -> list[int]:
118 + """
119 + Extract content token IDs from the input_ids.
120 +
121 + This extracts the non-reasoning content (e.g., everything after
122 + the </think> tag).
123 +
124 + Args:
125 + input_ids: The token IDs of the model output.
126 +
127 + Returns:
128 + The extracted content token IDs.
129 + """
130 +
131 + @abstractmethod
132 + def extract_reasoning(
133 + self,
134 + model_output: str,
135 + request: ChatCompletionRequest | ResponsesRequest,
136 + ) -> tuple[str | None, str | None]:
137 + """
138 + Extract reasoning content from a complete model-generated string.
139 +
140 + Used for non-streaming responses where we have the entire model
141 + response available before sending to the client.
142 +
143 + Args:
144 + model_output: The complete model-generated string.
145 + request: The request object used to generate the output.
146 +
147 + Returns:
148 + A tuple of (reasoning_content, response_content).
149 + """
150 +
151 + @abstractmethod
152 + def extract_reasoning_streaming(
153 + self,
154 + previous_text: str,
155 + current_text: str,
156 + delta_text: str,
157 + previous_token_ids: Sequence[int],
158 + current_token_ids: Sequence[int],
159 + delta_token_ids: Sequence[int],
160 + ) -> DeltaMessage | None:
161 + """
162 + Extract reasoning content from a streaming delta message.
163 +
164 + Args:
165 + previous_text: Text from all previous tokens.
166 + current_text: Text including the current delta.
167 + delta_text: The new text in this delta.
168 + previous_token_ids: Token IDs from previous generation.
169 + current_token_ids: All token IDs including current.
170 + delta_token_ids: The new token IDs in this delta.
171 +
172 + Returns:
173 + A DeltaMessage with reasoning and/or content fields, or None.
174 + """
175 +
176 + # ========== Tool Parser Methods ==========
177 +
178 + def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
179 + """
180 + Adjust the request parameters for tool calling.
181 +
182 + Can be overridden by subclasses to modify request parameters
183 + (e.g., setting structured output schemas for tool calling).
184 +
185 + Args:
186 + request: The original request.
187 +
188 + Returns:
189 + The adjusted request.
190 + """
191 + return request
192 +
193 + @abstractmethod
194 + def extract_tool_calls(
195 + self,
196 + model_output: str,
197 + request: ChatCompletionRequest,
198 + ) -> ExtractedToolCallInformation:
199 + """
200 + Extract tool calls from a complete model-generated string.
201 +
202 + Used for non-streaming responses.
203 +
204 + Args:
205 + model_output: The complete model-generated string.
206 + request: The request object used to generate the output.
207 +
208 + Returns:
209 + ExtractedToolCallInformation containing the tool calls.
210 + """
211 +
212 + @abstractmethod
213 + def extract_tool_calls_streaming(
214 + self,
215 + previous_text: str,
216 + current_text: str,
217 + delta_text: str,
218 + previous_token_ids: Sequence[int],
219 + current_token_ids: Sequence[int],
220 + delta_token_ids: Sequence[int],
221 + request: ChatCompletionRequest,
222 + ) -> DeltaMessage | None:
223 + """
224 + Extract tool calls from a streaming delta message.
225 +
226 + Args:
227 + previous_text: Text from all previous tokens.
228 + current_text: Text including the current delta.
229 + delta_text: The new text in this delta.
230 + previous_token_ids: Token IDs from previous generation.
231 + current_token_ids: All token IDs including current.
232 + delta_token_ids: The new token IDs in this delta.
233 + request: The request object.
234 +
235 + Returns:
236 + A DeltaMessage with tool_calls field, or None.
237 + """
238 +
239 +
240 +class DelegatingParser(Parser):
241 + """
242 + A Parser implementation that delegates to separate ReasoningParser and
243 + ToolParser instances.
244 +
245 + This is the recommended base class for creating model-specific parsers
246 + that combine existing reasoning and tool parser implementations.
247 + Subclasses should set `self._reasoning_parser` and `self._tool_parser`
248 + in their `__init__` method.
249 +
250 + If either parser is None, the corresponding methods will return default
251 + values (no reasoning extraction, no tool calls).
252 + """
253 +
254 + def extract_reasoning(
255 + self,
256 + model_output: str,
257 + request: ChatCompletionRequest | ResponsesRequest,
258 + ) -> tuple[str | None, str | None]:
259 + if self._reasoning_parser is None:
260 + return None, model_output
261 + return self._reasoning_parser.extract_reasoning(model_output, request)
262 +
263 + def extract_reasoning_streaming(
264 + self,
265 + previous_text: str,
266 + current_text: str,
267 + delta_text: str,
268 + previous_token_ids: Sequence[int],
269 + current_token_ids: Sequence[int],
270 + delta_token_ids: Sequence[int],
271 + ) -> DeltaMessage | None:
272 + if self._reasoning_parser is None:
273 + return DeltaMessage(content=delta_text)
274 + return self._reasoning_parser.extract_reasoning_streaming(
275 + previous_text,
276 + current_text,
277 + delta_text,
278 + previous_token_ids,
279 + current_token_ids,
280 + delta_token_ids,
281 + )
282 +
283 + def extract_tool_calls(
284 + self,
285 + model_output: str,
286 + request: ChatCompletionRequest,
287 + ) -> ExtractedToolCallInformation:
288 + if self._tool_parser is None:
289 + return ExtractedToolCallInformation(
290 + tools_called=False, tool_calls=[], content=model_output
291 + )
292 + return self._tool_parser.extract_tool_calls(model_output, request)
293 +
294 + def extract_tool_calls_streaming(
295 + self,
296 + previous_text: str,
297 + current_text: str,
298 + delta_text: str,
299 + previous_token_ids: Sequence[int],
300 + current_token_ids: Sequence[int],
301 + delta_token_ids: Sequence[int],
302 + request: ChatCompletionRequest,
303 + ) -> DeltaMessage | None:
304 + if self._tool_parser is None:
305 + return None
306 + return self._tool_parser.extract_tool_calls_streaming(
307 + previous_text,
308 + current_text,
309 + delta_text,
310 + previous_token_ids,
311 + current_token_ids,
312 + delta_token_ids,
313 + request,
314 + )
315 +
316 +
317 +class _WrappedParser(DelegatingParser):
318 + """
319 + A DelegatingParser subclass that instantiates parsers from class attributes.
320 +
321 + This class is used to dynamically create a parser that wraps individual
322 + ReasoningParser and ToolParser classes. The class attributes
323 + `reasoning_parser_cls` and `tool_parser_cls` should be set before
324 + instantiation.
325 +
326 + Usage:
327 + _WrappedParser.reasoning_parser_cls = MyReasoningParser
328 + _WrappedParser.tool_parser_cls = MyToolParser
329 + parser = _WrappedParser(tokenizer)
330 + """
331 +
332 + reasoning_parser_cls: type[ReasoningParser] | None = None
333 + tool_parser_cls: type[ToolParser] | None = None
334 +
335 + def __init__(self, tokenizer: TokenizerLike):
336 + super().__init__(tokenizer)
337 + # Instantiate the underlying parsers from class attributes
338 + if self.__class__.reasoning_parser_cls is not None:
339 + self._reasoning_parser = self.__class__.reasoning_parser_cls(tokenizer)
340 + if self.__class__.tool_parser_cls is not None:
341 + self._tool_parser = self.__class__.tool_parser_cls(tokenizer)
vllm/parser/minimax_m2_parser.py new
+52
@@ -0,0 +1,52 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +"""
5 +MiniMax M2 Parser - A unified parser for MiniMax M2 models.
6 +
7 +This parser combines the existing MiniMaxM2ReasoningParser and
8 +MinimaxM2ToolParser into a single unified interface by delegating
9 +to those implementations.
10 +"""
11 +
12 +from vllm.logger import init_logger
13 +from vllm.parser.abstract_parser import DelegatingParser
14 +from vllm.reasoning.minimax_m2_reasoning_parser import MiniMaxM2ReasoningParser
15 +from vllm.tokenizers import TokenizerLike
16 +from vllm.tool_parsers.minimax_m2_tool_parser import MinimaxM2ToolParser
17 +
18 +logger = init_logger(__name__)
19 +
20 +
21 +class MiniMaxM2Parser(DelegatingParser):
22 + """
23 + Unified parser for MiniMax M2 models that handles both reasoning
24 + extraction and tool call parsing.
25 +
26 + This parser delegates to the existing implementations:
27 + - MiniMaxM2ReasoningParser for reasoning extraction
28 + - MinimaxM2ToolParser for tool call parsing
29 +
30 + MiniMax M2 models have two special behaviors:
31 + 1. Reasoning: They don't generate <think> start token, only </think> end
32 + token. All content before </think> is reasoning, content after is the
33 + actual response.
34 + 2. Tool Calls: They use <minimax:tool_call>...</minimax:tool_call> tags
35 + with <invoke name="...">...</invoke> and <parameter name="...">...</parameter>
36 + syntax.
37 + """
38 +
39 + # Class-level parser classes for compatibility
40 + reasoning_parser_cls = MiniMaxM2ReasoningParser
41 + tool_parser_cls = MinimaxM2ToolParser
42 +
43 + def __init__(self, tokenizer: TokenizerLike):
44 + super().__init__(tokenizer)
45 +
46 + # Initialize the underlying parsers
47 + self._reasoning_parser = MiniMaxM2ReasoningParser(tokenizer)
48 + self._tool_parser = MinimaxM2ToolParser(tokenizer)
49 +
50 + logger.debug(
51 + "vLLM Successfully initialized parser %s!", self.__class__.__name__
52 + )
vllm/parser/parser_manager.py new
+308
@@ -0,0 +1,308 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from __future__ import annotations
5 +
6 +import importlib
7 +import os
8 +from collections.abc import Callable
9 +from typing import TYPE_CHECKING
10 +
11 +from vllm.logger import init_logger
12 +from vllm.utils.collection_utils import is_list_of
13 +from vllm.utils.import_utils import import_from_path
14 +
15 +if TYPE_CHECKING:
16 + from vllm.parser.abstract_parser import Parser
17 + from vllm.reasoning import ReasoningParser
18 + from vllm.tool_parsers import ToolParser
19 +
20 +logger = init_logger(__name__)
21 +
22 +
23 +class ParserManager:
24 + """
25 + Central registry for Parser implementations.
26 +
27 + Supports two registration modes:
28 + - Eager registration via `register_module`
29 + - Lazy registration via `register_lazy_module`
30 + """
31 +
32 + parsers: dict[str, type[Parser]] = {}
33 + lazy_parsers: dict[str, tuple[str, str]] = {} # name -> (module_path, class_name)
34 +
35 + @classmethod
36 + def get_parser_internal(cls, name: str) -> type[Parser]:
37 + """
38 + Retrieve a registered or lazily registered Parser class.
39 +
40 + Args:
41 + name: The registered name of the parser.
42 +
43 + Returns:
44 + The Parser class.
45 +
46 + Raises:
47 + KeyError: If no parser is found under the given name.
48 + """
49 + if name in cls.parsers:
50 + return cls.parsers[name]
51 +
52 + if name in cls.lazy_parsers:
53 + return cls._load_lazy_parser(name)
54 +
55 + registered = ", ".join(cls.list_registered())
56 + raise KeyError(f"Parser '{name}' not found. Available parsers: {registered}")
57 +
58 + @classmethod
59 + def _load_lazy_parser(cls, name: str) -> type[Parser]:
60 + """Import and register a lazily loaded parser."""
61 + from vllm.parser.abstract_parser import Parser
62 +
63 + module_path, class_name = cls.lazy_parsers[name]
64 + try:
65 + mod = importlib.import_module(module_path)
66 + parser_cls = getattr(mod, class_name)
67 + if not issubclass(parser_cls, Parser):
68 + raise TypeError(
69 + f"{class_name} in {module_path} is not a Parser subclass."
70 + )
71 + cls.parsers[name] = parser_cls # cache
72 + return parser_cls
73 + except Exception as e:
74 + logger.exception(
75 + "Failed to import lazy parser '%s' from %s: %s",
76 + name,
77 + module_path,
78 + e,
79 + )
80 + raise
81 +
82 + @classmethod
83 + def _register_module(
84 + cls,
85 + module: type[Parser],
86 + module_name: str | list[str] | None = None,
87 + force: bool = True,
88 + ) -> None:
89 + """Register a Parser class immediately."""
90 + from vllm.parser.abstract_parser import Parser
91 +
92 + if not issubclass(module, Parser):
93 + raise TypeError(
94 + f"module must be subclass of Parser, but got {type(module)}"
95 + )
96 +
97 + if module_name is None:
98 + module_names = [module.__name__]
99 + elif isinstance(module_name, str):
100 + module_names = [module_name]
101 + elif is_list_of(module_name, str):
102 + module_names = module_name
103 + else:
104 + raise TypeError("module_name must be str, list[str], or None.")
105 +
106 + for name in module_names:
107 + if not force and name in cls.parsers:
108 + existed = cls.parsers[name]
109 + raise KeyError(f"{name} is already registered at {existed.__module__}")
110 + cls.parsers[name] = module
111 +
112 + @classmethod
113 + def register_lazy_module(cls, name: str, module_path: str, class_name: str) -> None:
114 + """
115 + Register a lazy module mapping for delayed import.
116 +
117 + Example:
118 + ParserManager.register_lazy_module(
119 + name="minimax_m2",
120 + module_path="vllm.parser.minimax_m2_parser",
121 + class_name="MiniMaxM2Parser",
122 + )
123 + """
124 + cls.lazy_parsers[name] = (module_path, class_name)
125 +
126 + @classmethod
127 + def register_module(
128 + cls,
129 + name: str | list[str] | None = None,
130 + force: bool = True,
131 + module: type[Parser] | None = None,
132 + ) -> type[Parser] | Callable[[type[Parser]], type[Parser]]:
133 + """
134 + Register a Parser class.
135 +
136 + Can be used as a decorator or called directly.
137 +
138 + Usage:
139 + @ParserManager.register_module("my_parser")
140 + class MyParser(Parser):
141 + ...
142 +
143 + Or:
144 + ParserManager.register_module(module=MyParser)
145 + """
146 + if not isinstance(force, bool):
147 + raise TypeError(f"force must be a boolean, but got {type(force)}")
148 +
149 + # Immediate registration
150 + if module is not None:
151 + cls._register_module(module=module, module_name=name, force=force)
152 + return module
153 +
154 + # Decorator usage
155 + def _decorator(obj: type[Parser]) -> type[Parser]:
156 + module_path = obj.__module__
157 + class_name = obj.__name__
158 +
159 + if isinstance(name, str):
160 + names = [name]
161 + elif is_list_of(name, str):
162 + names = name
163 + else:
164 + names = [class_name]
165 +
166 + for n in names:
167 + cls.lazy_parsers[n] = (module_path, class_name)
168 +
169 + return obj
170 +
171 + return _decorator
172 +
173 + @classmethod
174 + def list_registered(cls) -> list[str]:
175 + """Return names of all registered parsers."""
176 + return sorted(set(cls.parsers.keys()) | set(cls.lazy_parsers.keys()))
177 +
178 + @classmethod
179 + def import_parser(cls, plugin_path: str) -> None:
180 + """Import a user-defined parser from an arbitrary path."""
181 + module_name = os.path.splitext(os.path.basename(plugin_path))[0]
182 + try:
183 + import_from_path(module_name, plugin_path)
184 + except Exception:
185 + logger.exception(
186 + "Failed to load module '%s' from %s.", module_name, plugin_path
187 + )
188 +
189 + @classmethod
190 + def get_tool_parser(
191 + cls,
192 + tool_parser_name: str | None = None,
193 + enable_auto_tools: bool = False,
194 + model_name: str | None = None,
195 + ) -> type[ToolParser] | None:
196 + """Get the tool parser based on the name."""
197 + from vllm.tool_parsers import ToolParserManager
198 +
199 + parser: type[ToolParser] | None = None
200 + if not enable_auto_tools or tool_parser_name is None:
201 + return parser
202 + logger.info('"auto" tool choice has been enabled.')
203 +
204 + try:
205 + if (
206 + tool_parser_name == "pythonic"
207 + and model_name
208 + and model_name.startswith("meta-llama/Llama-3.2")
209 + ):
210 + logger.warning(
211 + "Llama3.2 models may struggle to emit valid pythonic tool calls"
212 + )
213 + parser = ToolParserManager.get_tool_parser(tool_parser_name)
214 + except Exception as e:
215 + raise TypeError(
216 + "Error: --enable-auto-tool-choice requires "
217 + f"tool_parser:'{tool_parser_name}' which has not "
218 + "been registered"
219 + ) from e
220 + return parser
221 +
222 + @classmethod
223 + def get_reasoning_parser(
224 + cls,
225 + reasoning_parser_name: str | None,
226 + ) -> type[ReasoningParser] | None:
227 + """Get the reasoning parser based on the name."""
228 + from vllm.reasoning import ReasoningParserManager
229 +
230 + parser: type[ReasoningParser] | None = None
231 + if not reasoning_parser_name:
232 + return None
233 + try:
234 + parser = ReasoningParserManager.get_reasoning_parser(reasoning_parser_name)
235 + assert parser is not None
236 + except Exception as e:
237 + raise TypeError(f"{reasoning_parser_name=} has not been registered") from e
238 + return parser
239 +
240 + @classmethod
241 + def get_parser(
242 + cls,
243 + tool_parser_name: str | None = None,
244 + reasoning_parser_name: str | None = None,
245 + enable_auto_tools: bool = False,
246 + model_name: str | None = None,
247 + ) -> type[Parser] | None:
248 + """
249 + Get a unified Parser that handles both reasoning and tool parsing.
250 +
251 + This method checks if a unified Parser exists that can handle both
252 + reasoning extraction and tool call parsing. If no unified parser
253 + exists, it creates a DelegatingParser that wraps the individual
254 + reasoning and tool parsers.
255 +
256 + Args:
257 + tool_parser_name: The name of the tool parser.
258 + reasoning_parser_name: The name of the reasoning parser.
259 + enable_auto_tools: Whether auto tool choice is enabled.
260 + model_name: The model name for parser-specific warnings.
261 +
262 + Returns:
263 + A Parser class, or None if neither parser is specified.
264 + """
265 + from vllm.parser.abstract_parser import _WrappedParser
266 +
267 + if not tool_parser_name and not reasoning_parser_name:
268 + return None
269 +
270 + # Strategy 1: If both names match, check for a unified parser with that name
271 + if tool_parser_name and tool_parser_name == reasoning_parser_name:
272 + try:
273 + parser = cls.get_parser_internal(tool_parser_name)
274 + logger.info(
275 + "Using unified parser '%s' for both reasoning and tool parsing.",
276 + tool_parser_name,
277 + )
278 + return parser
279 + except KeyError:
280 + pass # No unified parser with this name
281 +
282 + # Strategy 2: Check for parser with either name
283 + for name in [tool_parser_name, reasoning_parser_name]:
284 + if name:
285 + try:
286 + parser = cls.get_parser_internal(name)
287 + logger.info(
288 + "Using unified parser '%s' for reasoning and tool parsing.",
289 + name,
290 + )
291 + return parser
292 + except KeyError:
293 + pass
294 +
295 + # Strategy 3: Create a DelegatingParser with the individual parser classes
296 + reasoning_parser_cls = cls.get_reasoning_parser(reasoning_parser_name)
297 + tool_parser_cls = cls.get_tool_parser(
298 + tool_parser_name, enable_auto_tools, model_name
299 + )
300 +
301 + if reasoning_parser_cls is None and tool_parser_cls is None:
302 + return None
303 +
304 + # Set the class-level attributes on the imported _WrappedParser
305 + _WrappedParser.reasoning_parser_cls = reasoning_parser_cls
306 + _WrappedParser.tool_parser_cls = tool_parser_cls
307 +
308 + return _WrappedParser