A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[Refactor][Frontend] Keep all logic about reasoning into one class (#14428)
Signed-off-by: Ce Gao <cegao@tensorchord.ai>
Ce Gao committed
Mar 28, 2025 at 15:23 UTC
32b14baf8a1f7195ca09484de3008063569b43c5
18 files changed
+171
-200
tests/reasoning/__init__.py
renamed
tests/reasoning/test_deepseekr1_reasoning_parser.py
renamed
+41
-11
@@ -3,74 +3,92 @@
3
import pytest
4
from transformers import AutoTokenizer
5
6
-from tests.entrypoints.openai.reasoning_parsers.utils import (
7
- run_reasoning_extraction)
8
-from vllm.entrypoints.openai.reasoning_parsers import (ReasoningParser,
9
- ReasoningParserManager)
6
+from tests.reasoning.utils import run_reasoning_extraction
7
+from vllm.reasoning import ReasoningParser, ReasoningParserManager
8
9
parser_name = "deepseek_r1"
10
start_token = "<think>"
11
end_token = "</think>"
12
13
+REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
14
+
15
+
16
+@pytest.fixture(scope="module")
17
+def deepseek_r1_qwen_tokenizer():
18
+ return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
19
+
20
+
21
SIMPLE_REASONING = {
22
"output": "This is a reasoning section</think>This is the rest",
23
"reasoning_content": "This is a reasoning section",
24
"content": "This is the rest",
25
+ "is_reasoning_end": True,
26
}
27
COMPLETE_REASONING = {
28
"output": "This is a reasoning section</think>",
29
"reasoning_content": "This is a reasoning section",
30
"content": None,
31
+ "is_reasoning_end": True,
32
}
33
NO_CONTENT = {
34
"output": "This is content",
35
"reasoning_content": "This is content",
36
"content": None,
37
+ "is_reasoning_end": False,
38
}
39
NO_REASONING_STREAMING = {
40
"output": "This is a reasoning section",
41
"reasoning_content": "This is a reasoning section",
42
"content": None,
43
+ "is_reasoning_end": False,
44
}
45
MULTIPLE_LINES = {
46
"output": "This\nThat</think>This is the rest\nThat",
47
"reasoning_content": "This\nThat",
48
"content": "This is the rest\nThat",
49
+ "is_reasoning_end": True,
50
}
51
SHORTEST_REASONING_NO_STREAMING = {
52
"output": "</think>This is the rest",
53
"reasoning_content": "",
54
"content": "This is the rest",
55
+ "is_reasoning_end": True,
56
}
57
SHORTEST_REASONING = {
58
"output": "</think>This is the rest",
59
"reasoning_content": None,
60
"content": "This is the rest",
61
+ "is_reasoning_end": True,
62
}
63
REASONING_WITH_THINK = {
64
"output": "<think>This is a reasoning section</think>This is the rest",
65
"reasoning_content": "This is a reasoning section",
66
"content": "This is the rest",
67
+ "is_reasoning_end": True,
68
}
69
COMPLETE_REASONING_WITH_THINK = {
70
"output": "<think>This is a reasoning section</think>",
71
"reasoning_content": "This is a reasoning section",
72
"content": None,
73
+ "is_reasoning_end": True,
74
}
75
MULTIPLE_LINES_WITH_THINK = {
76
"output": "<think>This\nThat</think>This is the rest\nThat",
77
"reasoning_content": "This\nThat",
78
"content": "This is the rest\nThat",
79
+ "is_reasoning_end": True,
80
}
81
SHORTEST_REASONING_NO_STREAMING_WITH_THINK = {
82
"output": "</think>This is the rest",
83
"reasoning_content": "",
84
"content": "This is the rest",
85
+ "is_reasoning_end": True,
86
}
87
SHORTEST_REASONING_WITH_THINK = {
88
"output": "</think>This is the rest",
89
"reasoning_content": None,
90
"content": "This is the rest",
91
+ "is_reasoning_end": True,
92
}
93
94
TEST_CASES = [
@@ -166,23 +184,21 @@ TEST_CASES = [
184
),
185
]
186
169
-# Global tokenizer initialization to avoid repeated loading
170
-tokenizer = AutoTokenizer.from_pretrained("facebook/opt-125m")
171
-tokenizer.add_tokens([start_token, end_token])
172
-
187
188
@pytest.mark.parametrize("streaming, param_dict", TEST_CASES)
189
def test_reasoning(
190
streaming: bool,
191
param_dict: dict,
192
+ deepseek_r1_qwen_tokenizer,
193
):
179
- output = tokenizer.tokenize(param_dict["output"])
194
+ output = deepseek_r1_qwen_tokenizer.tokenize(param_dict["output"])
195
# decode everything to tokens
196
output_tokens: list[str] = [
182
- tokenizer.convert_tokens_to_string([token]) for token in output
197
+ deepseek_r1_qwen_tokenizer.convert_tokens_to_string([token])
198
+ for token in output
199
]
200
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(
185
- parser_name)(tokenizer)
201
+ parser_name)(deepseek_r1_qwen_tokenizer)
202
203
reasoning, content = run_reasoning_extraction(parser,
204
output_tokens,
@@ -190,3 +206,17 @@ def test_reasoning(
206
207
assert reasoning == param_dict["reasoning_content"]
208
assert content == param_dict["content"]
209
+
210
+ # Test is_reasoning_end
211
+ output_ids = deepseek_r1_qwen_tokenizer.convert_tokens_to_ids(output)
212
+ is_reasoning_end = parser.is_reasoning_end(output_ids)
213
+ assert is_reasoning_end == param_dict["is_reasoning_end"]
214
+
215
+ # Test extract_content
216
+ if param_dict["content"] is not None:
217
+ content = parser.extract_content_ids(output_ids)
218
+ assert content == deepseek_r1_qwen_tokenizer.convert_tokens_to_ids(
219
+ deepseek_r1_qwen_tokenizer.tokenize(param_dict["content"]))
220
+ else:
221
+ content = parser.extract_content_ids(output)
222
+ assert content == []
tests/reasoning/test_granite_reasoning_parser.py
renamed
+2
-4
@@ -2,10 +2,8 @@
2
import pytest
3
from transformers import AutoTokenizer
4
5
-from tests.entrypoints.openai.reasoning_parsers.utils import (
6
- DeltaMessage, run_reasoning_extraction)
7
-from vllm.entrypoints.openai.reasoning_parsers import (ReasoningParser,
8
- ReasoningParserManager)
5
+from tests.reasoning.utils import DeltaMessage, run_reasoning_extraction
6
+from vllm.reasoning import ReasoningParser, ReasoningParserManager
7
8
parser_name = "granite"
9
START_REASONING = "Here is my thought process:"
tests/reasoning/utils.py
renamed
+1
-1
@@ -4,7 +4,7 @@ from typing import Optional, Union
4
5
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
6
DeltaMessage)
7
-from vllm.entrypoints.openai.reasoning_parsers import ReasoningParser
7
+from vllm.reasoning import ReasoningParser
8
9
10
class StreamingReasoningReconstructor:
vllm/engine/arg_utils.py
+2
-1
@@ -23,6 +23,7 @@ from vllm.executor.executor_base import ExecutorBase
23
from vllm.logger import init_logger
24
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
25
from vllm.plugins import load_general_plugins
26
+from vllm.reasoning import ReasoningParserManager
27
from vllm.test_utils import MODEL_WEIGHTS_S3_BUCKET, MODELS_ON_S3
28
from vllm.transformers_utils.utils import check_gguf_file
29
from vllm.usage.usage_lib import UsageContext
@@ -1119,7 +1120,7 @@ class EngineArgs:
1120
parser.add_argument(
1121
"--reasoning-parser",
1122
type=str,
1122
- choices=["deepseek_r1", "granite"],
1123
+ choices=list(ReasoningParserManager.reasoning_parsers),
1124
default=None,
1125
help=
1126
"Select the reasoning parser depending on the model that you're "
vllm/engine/llm_engine.py
+3
-2
@@ -2080,8 +2080,9 @@ class LLMEngine:
2080
guided_decoding.backend = guided_decoding.backend or \
2081
self.decoding_config.guided_decoding_backend
2082
2083
- logger.debug("Reasoning backend: %s",
2084
- self.decoding_config.reasoning_backend)
2083
+ if self.decoding_config.reasoning_backend is not None:
2084
+ logger.debug("Building with reasoning backend %s",
2085
+ self.decoding_config.reasoning_backend)
2086
2087
processor = get_local_guided_decoding_logits_processor(
2088
guided_params=guided_decoding,
vllm/entrypoints/openai/api_server.py
+1
-1
@@ -68,7 +68,6 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
68
TranscriptionRequest,
69
TranscriptionResponse,
70
UnloadLoRAAdapterRequest)
71
-from vllm.entrypoints.openai.reasoning_parsers import ReasoningParserManager
71
# yapf: enable
72
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
73
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
@@ -85,6 +84,7 @@ from vllm.entrypoints.openai.serving_transcription import (
84
from vllm.entrypoints.openai.tool_parsers import ToolParserManager
85
from vllm.entrypoints.utils import load_aware_call, with_cancellation
86
from vllm.logger import init_logger
87
+from vllm.reasoning import ReasoningParserManager
88
from vllm.transformers_utils.config import (
89
maybe_register_config_serialize_by_value)
90
from vllm.transformers_utils.tokenizer import MistralTokenizer
vllm/entrypoints/openai/serving_chat.py
+1
-2
@@ -23,8 +23,6 @@ from vllm.entrypoints.openai.protocol import (
23
ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
24
DeltaToolCall, ErrorResponse, FunctionCall, PromptTokenUsageInfo,
25
RequestResponseMetadata, ToolCall, UsageInfo)
26
-from vllm.entrypoints.openai.reasoning_parsers import (ReasoningParser,
27
- ReasoningParserManager)
26
from vllm.entrypoints.openai.serving_engine import (OpenAIServing,
27
clamp_prompt_logprobs)
28
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
@@ -33,6 +31,7 @@ from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import (
31
MistralToolCall)
32
from vllm.logger import init_logger
33
from vllm.outputs import CompletionOutput, RequestOutput
34
+from vllm.reasoning import ReasoningParser, ReasoningParserManager
35
from vllm.sampling_params import BeamSearchParams, SamplingParams
36
from vllm.sequence import Logprob
37
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
vllm/model_executor/guided_decoding/__init__.py
+11
-4
@@ -5,10 +5,10 @@ from __future__ import annotations
5
from typing import TYPE_CHECKING
6
7
from vllm.logger import init_logger
8
-from vllm.model_executor.guided_decoding.reasoner import get_reasoner
8
from vllm.model_executor.guided_decoding.utils import (
9
convert_lark_to_gbnf, grammar_is_likely_lark,
10
has_lmf_unsupported_json_features, has_xgrammar_unsupported_json_features)
11
+from vllm.reasoning import ReasoningParserManager
12
13
if TYPE_CHECKING:
14
from transformers import PreTrainedTokenizer
@@ -107,7 +107,11 @@ async def get_guided_decoding_logits_processor(
107
model_config: ModelConfig,
108
reasoning_backend: str | None = None) -> LogitsProcessor | None:
109
110
- reasoner = get_reasoner(tokenizer, reasoning_backend)
110
+ reasoner = None
111
+ if reasoning_backend is not None:
112
+ reasoner_class = ReasoningParserManager.get_reasoning_parser(
113
+ reasoning_backend)
114
+ reasoner = reasoner_class(tokenizer)
115
116
guided_params = maybe_backend_fallback(guided_params)
117
@@ -146,8 +150,11 @@ def get_local_guided_decoding_logits_processor(
150
reasoning_backend: str | None = None) -> LogitsProcessor | None:
151
guided_params = maybe_backend_fallback(guided_params)
152
149
- # Get the reasoner if needed, it will be None if reasoning_
150
- reasoner = get_reasoner(tokenizer, reasoning_backend)
153
+ reasoner = None
154
+ if reasoning_backend is not None:
155
+ reasoner_class = ReasoningParserManager.get_reasoning_parser(
156
+ reasoning_backend)
157
+ reasoner = reasoner_class(tokenizer)
158
159
# CFG grammar not supported by LMFE, so we use outlines instead
160
if guided_params.backend_name == 'outlines':
vllm/model_executor/guided_decoding/outlines_decoding.py
+4
-4
@@ -12,7 +12,7 @@ from transformers import PreTrainedTokenizerBase
12
13
from vllm.model_executor.guided_decoding.outlines_logits_processors import (
14
CFGLogitsProcessor, JSONLogitsProcessor, RegexLogitsProcessor)
15
-from vllm.model_executor.guided_decoding.reasoner import Reasoner
15
+from vllm.reasoning import ReasoningParser
16
from vllm.sampling_params import GuidedDecodingParams
17
18
@@ -61,7 +61,7 @@ _MAX_THREADPOOL_WORKERS = 16
61
async def get_outlines_guided_decoding_logits_processor(
62
guided_params: GuidedDecodingParams,
63
tokenizer: PreTrainedTokenizerBase,
64
- reasoner: Optional[Reasoner],
64
+ reasoner: Optional[ReasoningParser],
65
) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor,
66
None]:
67
"""
@@ -92,7 +92,7 @@ async def get_outlines_guided_decoding_logits_processor(
92
def get_local_outlines_guided_decoding_logits_processor(
93
guided_params: GuidedDecodingParams,
94
tokenizer: PreTrainedTokenizerBase,
95
- reasoner: Optional[Reasoner],
95
+ reasoner: Optional[ReasoningParser],
96
) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor,
97
None]:
98
"""
@@ -141,7 +141,7 @@ def _get_logits_processor(
141
tokenizer: PreTrainedTokenizerBase,
142
mode: GuidedDecodingMode,
143
whitespace_pattern: Union[str, None],
144
- reasoner: Optional[Reasoner],
144
+ reasoner: Optional[ReasoningParser],
145
) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor]:
146
if mode == GuidedDecodingMode.JSON:
147
return JSONLogitsProcessor(guide, tokenizer, whitespace_pattern,
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+7
-7
@@ -34,8 +34,8 @@ from transformers import PreTrainedTokenizerBase
34
35
import vllm.envs as envs
36
from vllm.logger import init_logger
37
-from vllm.model_executor.guided_decoding.reasoner import Reasoner
37
from vllm.platforms import current_platform
38
+from vllm.reasoning import ReasoningParser
39
40
logger = init_logger(__name__)
41
@@ -49,9 +49,9 @@ else:
49
50
class BaseLogitsProcessor:
51
52
- def __init__(self, guide: Guide, reasoner: Optional[Reasoner]):
52
+ def __init__(self, guide: Guide, reasoner: Optional[ReasoningParser]):
53
self._guide: Guide = guide
54
- self._reasoner: Optional[Reasoner] = reasoner
54
+ self._reasoner: Optional[ReasoningParser] = reasoner
55
# CFGState is used for the FSM state for CFGGuide
56
self._fsm_state: DefaultDict[int, Union[int,
57
CFGState]] = defaultdict(int)
@@ -69,7 +69,7 @@ class BaseLogitsProcessor:
69
# Remove the reasoning tokens from the input_ids
70
# We need this because our implementation relies on the
71
# hash of the input_ids to store the FSM state.
72
- input_ids = self._reasoner.extract_content(input_ids)
72
+ input_ids = self._reasoner.extract_content_ids(input_ids)
73
74
seq_id = hash(tuple(input_ids))
75
@@ -142,7 +142,7 @@ class RegexLogitsProcessor(BaseLogitsProcessor):
142
self,
143
regex_string: str,
144
tokenizer: PreTrainedTokenizerBase,
145
- reasoner: Optional[Reasoner],
145
+ reasoner: Optional[ReasoningParser],
146
):
147
"""Compile the FSM that drives the regex-structured generation.
148
@@ -163,7 +163,7 @@ class JSONLogitsProcessor(RegexLogitsProcessor):
163
def __init__(self, schema: Union[str, Dict, BaseModel],
164
tokenizer: PreTrainedTokenizerBase,
165
whitespace_pattern: Union[str, None],
166
- reasoner: Optional[Reasoner]):
166
+ reasoner: Optional[ReasoningParser]):
167
"""Compile the FSM that drives the JSON-guided generation.
168
169
Parameters
@@ -203,7 +203,7 @@ class CFGLogitsProcessor(BaseLogitsProcessor):
203
return CFGGuide(cfg, tokenizer)
204
205
def __init__(self, cfg: str, tokenizer: PreTrainedTokenizerBase,
206
- reasoner: Optional[Reasoner]):
206
+ reasoner: Optional[ReasoningParser]):
207
"""Compile the FSM that drives the context free grammar generation.
208
209
Parameters
vllm/model_executor/guided_decoding/reasoner/deepseek_reasoner.py
deleted
-38
@@ -1,38 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-from dataclasses import dataclass
3
-
4
-from transformers import PreTrainedTokenizer
5
-
6
-from vllm.model_executor.guided_decoding.reasoner.reasoner import Reasoner
7
-
8
-
9
-@dataclass
10
-class DeepSeekReasoner(Reasoner):
11
- """
12
- Reasoner for DeepSeek R series models.
13
- """
14
- start_token_id: int
15
- end_token_id: int
16
-
17
- start_token: str = "<think>"
18
- end_token: str = "</think>"
19
-
20
- @classmethod
21
- def from_tokenizer(cls, tokenizer: PreTrainedTokenizer) -> Reasoner:
22
- return cls(start_token_id=tokenizer.encode(
23
- "<think>", add_special_tokens=False)[0],
24
- end_token_id=tokenizer.encode("</think>",
25
- add_special_tokens=False)[0])
26
-
27
- def is_reasoning_end(self, input_ids: list[int]) -> bool:
28
- return self.end_token_id in input_ids
29
-
30
- def extract_content(self, input_ids: list[int]) -> list[int]:
31
- """
32
- Extract the content after the end tokens
33
- """
34
- if self.end_token_id not in input_ids or \
35
- input_ids.index(self.end_token_id) + 1 == len(input_ids):
36
- return []
37
- else:
38
- return input_ids[input_ids.index(self.end_token_id) + 1:]
vllm/model_executor/guided_decoding/reasoner/reasoner.py
deleted
-23
@@ -1,23 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-from __future__ import annotations
3
-
4
-from abc import ABC, abstractmethod
5
-from dataclasses import dataclass
6
-
7
-from transformers import PreTrainedTokenizer
8
-
9
-
10
-@dataclass
11
-class Reasoner(ABC):
12
-
13
- @abstractmethod
14
- def from_tokenizer(cls, tokenizer: PreTrainedTokenizer) -> Reasoner:
15
- pass
16
-
17
- @abstractmethod
18
- def is_reasoning_end(self, input_ids: list[int]) -> bool:
19
- pass
20
-
21
- @abstractmethod
22
- def extract_content(self, input_ids: list[int]) -> list[int]:
23
- pass
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+3
-3
@@ -27,7 +27,7 @@ if TYPE_CHECKING:
27
from transformers import PreTrainedTokenizer
28
29
from vllm.config import ModelConfig
30
- from vllm.model_executor.guided_decoding.reasoner import Reasoner
30
+ from vllm.reasoning import ReasoningParser
31
from vllm.sampling_params import GuidedDecodingParams
32
33
logger = init_logger(__name__)
@@ -37,7 +37,7 @@ def get_local_xgrammar_guided_decoding_logits_processor(
37
guided_params: GuidedDecodingParams,
38
tokenizer: PreTrainedTokenizer,
39
model_config: ModelConfig,
40
- reasoner: Reasoner | None,
40
+ reasoner: ReasoningParser | None,
41
max_threads: int = 8):
42
config = GrammarConfig.from_guided_params(guided_params=guided_params,
43
model_config=model_config,
@@ -280,7 +280,7 @@ class GrammarConfig:
280
class XGrammarLogitsProcessor:
281
"""Wrapper class to support pickle protocol"""
282
config: GrammarConfig
283
- reasoner: Reasoner | None = None
283
+ reasoner: ReasoningParser | None = None
284
285
ctx: xgr.CompiledGrammar | None = None
286
tokenizer_info: xgr.TokenizerInfo = None # type: ignore[assignment]
vllm/reasoning/__init__.py
renamed
vllm/reasoning/abs_reasoning_parsers.py
renamed
+47
-54
@@ -17,7 +17,7 @@ logger = init_logger(__name__)
17
18
class ReasoningParser:
19
"""
20
- Abstract reasoning parser class that should not be used directly.
20
+ Abstract reasoning parser class that should not be used directly.
21
Provided and methods should be used in derived classes.
22
23
It is used to extract reasoning content from the model output.
@@ -32,6 +32,36 @@ class ReasoningParser:
32
# whereas all tokenizers have .get_vocab()
33
return self.model_tokenizer.get_vocab()
34
35
+ @abstractmethod
36
+ def is_reasoning_end(self, input_ids: list[int]) -> bool:
37
+ """
38
+ Check if the reasoning content ends in the input_ids.
39
+
40
+ It is used in structured engines like `xgrammar` to check if the
41
+ reasoning content ends in the model output.
42
+
43
+ Parameters:
44
+ input_ids: list[int]
45
+ The input_ids of the model output.
46
+
47
+ Returns:
48
+ bool
49
+ True if the reasoning content ends in the input_ids.
50
+ """
51
+
52
+ @abstractmethod
53
+ def extract_content_ids(self, input_ids: list[int]) -> list[int]:
54
+ """
55
+ Extract content token ids from the input_ids.
56
+ Parameters:
57
+ input_ids: list[int]
58
+ The input_ids of the model output.
59
+ Returns:
60
+ list[int]
61
+ The extracted content from the input_ids.
62
+ """
63
+
64
+ @abstractmethod
65
def extract_reasoning_content(
66
self, model_output: str, request: ChatCompletionRequest
67
) -> tuple[Optional[str], Optional[str]]:
@@ -53,10 +83,7 @@ class ReasoningParser:
83
A tuple containing the reasoning content and the content.
84
"""
85
56
- raise NotImplementedError(
57
- "AbstractReasoningParser.extract_reasoning_calls "
58
- "has not been implemented!")
59
-
86
+ @abstractmethod
87
def extract_reasoning_content_streaming(
88
self,
89
previous_text: str,
@@ -73,43 +100,6 @@ class ReasoningParser:
100
the current tokens/diffs, but also the information about what has
101
previously been parsed and extracted (see constructor)
102
"""
76
- raise NotImplementedError(
77
- "AbstractReasoningParser.extract_reasoning_content_streaming "
78
- "has not been implemented!")
79
-
80
- # TODO: need to rebase by PR #14428
81
- @abstractmethod
82
- def is_reasoning_end(self, input_ids: list[int]) -> bool:
83
- """
84
- Check if the reasoning content ends in the input_ids.
85
- Parameters:
86
- input_ids: list[int]
87
- The input_ids of the model output.
88
- Returns:
89
- bool
90
- True if the reasoning content ends in the input_ids.
91
- """
92
-
93
- raise NotImplementedError(
94
- "AbstractReasoningParser.is_reasoning_end has"
95
- "not been implemented!")
96
-
97
- # TODO: need to rebase by PR #14428
98
- @abstractmethod
99
- def extract_content_ids(self, input_ids: list[int]) -> list[int]:
100
- """
101
- Extract content token ids from the input_ids.
102
- Parameters:
103
- input_ids: list[int]
104
- The input_ids of the model output.
105
- Returns:
106
- list[int]
107
- The extracted content from the input_ids.
108
- """
109
-
110
- raise NotImplementedError(
111
- "AbstractReasoningParser.extract_content_ids has"
112
- " not been implemented!")
103
104
105
class ReasoningParserManager:
@@ -125,14 +115,16 @@ class ReasoningParserManager:
115
if name in cls.reasoning_parsers:
116
return cls.reasoning_parsers[name]
117
128
- raise KeyError(f"reasoning helper: '{name}' not found in "
129
- "reasoning_parsers")
118
+ raise KeyError(
119
+ f"reasoning helper: '{name}' not found in reasoning_parsers")
120
121
@classmethod
132
- def _register_module(cls,
133
- module: type,
134
- module_name: Optional[Union[str, list[str]]] = None,
135
- force: bool = True) -> None:
122
+ def _register_module(
123
+ cls,
124
+ module: type,
125
+ module_name: Optional[Union[str, list[str]]] = None,
126
+ force: bool = True,
127
+ ) -> None:
128
if not issubclass(module, ReasoningParser):
129
raise TypeError("module must be subclass of ReasoningParser, "
130
f"but got {type(module)}")
@@ -149,13 +141,14 @@ class ReasoningParserManager:
141
142
@classmethod
143
def register_module(
152
- cls,
153
- name: Optional[Union[str, list[str]]] = None,
154
- force: bool = True,
155
- module: Union[type, None] = None) -> Union[type, Callable]:
144
+ cls,
145
+ name: Optional[Union[str, list[str]]] = None,
146
+ force: bool = True,
147
+ module: Union[type, None] = None,
148
+ ) -> Union[type, Callable]:
149
"""
150
Register module with the given name or name list. it can be used as a
158
- decoder(with module as None) or normal function(with module as not
151
+ decoder(with module as None) or normal function(with module as not
152
None).
153
"""
154
if not isinstance(force, bool):
@@ -183,7 +176,7 @@ class ReasoningParserManager:
176
@classmethod
177
def import_reasoning_parser(cls, plugin_path: str) -> None:
178
"""
186
- Import a user-defined reasoning parser by the path
179
+ Import a user-defined reasoning parser by the path
180
of the reasoning parser define file.
181
"""
182
module_name = os.path.splitext(os.path.basename(plugin_path))[0]
vllm/reasoning/deepseek_r1_reasoning_parser.py
renamed
+47
-43
@@ -8,9 +8,8 @@ from transformers import PreTrainedTokenizerBase
8
9
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
10
DeltaMessage)
11
-from vllm.entrypoints.openai.reasoning_parsers.abs_reasoning_parsers import (
12
- ReasoningParser, ReasoningParserManager)
11
from vllm.logger import init_logger
12
+from vllm.reasoning import ReasoningParser, ReasoningParserManager
13
14
logger = init_logger(__name__)
15
@@ -20,43 +19,45 @@ class DeepSeekR1ReasoningParser(ReasoningParser):
19
"""
20
Reasoning parser for DeepSeek R1 model.
21
23
- The DeepSeek R1 model uses <think>...</think> tokens to denote reasoning
22
+ The DeepSeek R1 model uses <think>...</think> tokens to denote reasoning
23
text. This parser extracts the reasoning content from the model output.
24
"""
25
26
+ start_token_id: int
27
+ end_token_id: int
28
+
29
+ start_token: str = "<think>"
30
+ end_token: str = "</think>"
31
+
32
def __init__(self, tokenizer: PreTrainedTokenizerBase):
33
super().__init__(tokenizer)
29
- self.think_start_token = "<think>"
30
- self.think_end_token = "</think>"
34
35
self.reasoning_regex = re.compile(
33
- rf"{self.think_start_token}(.*?){self.think_end_token}", re.DOTALL)
36
+ rf"{self.start_token}(.*?){self.end_token}", re.DOTALL)
37
38
if not self.model_tokenizer:
39
raise ValueError(
40
"The model tokenizer must be passed to the ReasoningParser "
41
"constructor during construction.")
42
40
- self.think_start_token_id = self.vocab.get(self.think_start_token)
41
- self.think_end_token_id = self.vocab.get(self.think_end_token)
42
- if (self.think_start_token_id is None
43
- or self.think_end_token_id is None):
43
+ self.start_token_id = self.vocab.get(self.start_token)
44
+ self.end_token_id = self.vocab.get(self.end_token)
45
+ if self.start_token_id is None or self.end_token_id is None:
46
raise RuntimeError(
47
"DeepSeek R1 reasoning parser could not locate think start/end "
48
"tokens in the tokenizer!")
49
48
- # TODO: need to rebase by PR #14428
50
def is_reasoning_end(self, input_ids: list[int]) -> bool:
50
- return self.think_end_token_id in input_ids
51
+ return self.end_token_id in input_ids
52
53
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
54
"""
55
Extract the content after the end tokens
56
"""
56
- if self.think_end_token_id not in input_ids[:-1]:
57
+ if self.end_token_id not in input_ids[:-1]:
58
return []
59
else:
59
- return input_ids[input_ids.index(self.think_end_token_id) + 1:]
60
+ return input_ids[input_ids.index(self.end_token_id) + 1:]
61
62
def extract_reasoning_content_streaming(
63
self,
@@ -77,22 +78,24 @@ class DeepSeekR1ReasoningParser(ReasoningParser):
78
"""
79
# Skip single special tokens
80
if len(delta_token_ids) == 1 and (delta_token_ids[0] in [
80
- self.think_start_token_id, self.think_end_token_id
81
+ self.start_token_id, self.end_token_id
82
]):
83
return None
84
85
# Check if <think> is present in previous or delta.
86
# Keep compatibility with models that don't generate <think> tokens.
86
- if self.think_start_token_id in previous_token_ids:
87
- if self.think_end_token_id in delta_token_ids:
87
+ if self.start_token_id in previous_token_ids:
88
+ if self.end_token_id in delta_token_ids:
89
# <think> in previous, </think> in delta,
90
# extract reasoning content
90
- end_index = delta_text.find(self.think_end_token)
91
+ end_index = delta_text.find(self.end_token)
92
reasoning_content = delta_text[:end_index]
92
- content = delta_text[end_index + len(self.think_end_token):]
93
- return DeltaMessage(reasoning_content=reasoning_content,
94
- content=content if content else None)
95
- elif self.think_end_token_id in previous_token_ids:
93
+ content = delta_text[end_index + len(self.end_token):]
94
+ return DeltaMessage(
95
+ reasoning_content=reasoning_content,
96
+ content=content if content else None,
97
+ )
98
+ elif self.end_token_id in previous_token_ids:
99
# <think> in previous, </think> in previous,
100
# reasoning content continues
101
return DeltaMessage(content=delta_text)
@@ -100,17 +103,18 @@ class DeepSeekR1ReasoningParser(ReasoningParser):
103
# <think> in previous, no </think> in previous or delta,
104
# reasoning content continues
105
return DeltaMessage(reasoning_content=delta_text)
103
- elif self.think_start_token_id in delta_token_ids:
104
- if self.think_end_token_id in delta_token_ids:
106
+ elif self.start_token_id in delta_token_ids:
107
+ if self.end_token_id in delta_token_ids:
108
# <think> in delta, </think> in delta, extract reasoning content
106
- start_index = delta_text.find(self.think_start_token)
107
- end_index = delta_text.find(self.think_end_token)
109
+ start_index = delta_text.find(self.start_token)
110
+ end_index = delta_text.find(self.end_token)
111
reasoning_content = delta_text[start_index +
109
- len(self.think_start_token
110
- ):end_index]
111
- content = delta_text[end_index + len(self.think_end_token):]
112
- return DeltaMessage(reasoning_content=reasoning_content,
113
- content=content if content else None)
112
+ len(self.start_token):end_index]
113
+ content = delta_text[end_index + len(self.end_token):]
114
+ return DeltaMessage(
115
+ reasoning_content=reasoning_content,
116
+ content=content if content else None,
117
+ )
118
else:
119
# <think> in delta, no </think> in delta,
120
# reasoning content continues
@@ -119,15 +123,17 @@ class DeepSeekR1ReasoningParser(ReasoningParser):
123
# No <think> in previous or delta, also need to check for </think>.
124
# Because the model may have generated </think> without <think>
125
# Ref https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f
122
- if self.think_end_token_id in delta_token_ids:
126
+ if self.end_token_id in delta_token_ids:
127
# </think> in delta with more tokens,
128
# extract reasoning content and content
125
- end_index = delta_text.find(self.think_end_token)
129
+ end_index = delta_text.find(self.end_token)
130
reasoning_content = delta_text[:end_index]
127
- content = delta_text[end_index + len(self.think_end_token):]
128
- return DeltaMessage(reasoning_content=reasoning_content,
129
- content=content if content else None)
130
- elif self.think_end_token_id in previous_token_ids:
131
+ content = delta_text[end_index + len(self.end_token):]
132
+ return DeltaMessage(
133
+ reasoning_content=reasoning_content,
134
+ content=content if content else None,
135
+ )
136
+ elif self.end_token_id in previous_token_ids:
137
# </think> in previous, thinking content ends
138
return DeltaMessage(content=delta_text)
139
else:
@@ -137,22 +143,20 @@ class DeepSeekR1ReasoningParser(ReasoningParser):
143
def extract_reasoning_content(
144
self, model_output: str, request: ChatCompletionRequest
145
) -> tuple[Optional[str], Optional[str]]:
140
-
146
# DeepSeek R1 doesn't generate <think> now.
147
# Thus we assume the reasoning content is always at the start.
148
# Ref https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f
144
- if self.think_end_token not in model_output:
149
+ if self.end_token not in model_output:
150
return model_output, None
151
else:
152
# Add a start token if it's missing to keep compatibility.
148
- if self.think_start_token not in model_output:
149
- model_output = f"{self.think_start_token}{model_output}"
153
+ if self.start_token not in model_output:
154
+ model_output = f"{self.start_token}{model_output}"
155
# Use a regex to find the reasoning content
156
reasoning_content = self.reasoning_regex.findall(model_output)[0]
157
158
end_index = len(
154
- f"{self.think_start_token}{reasoning_content}{self.think_end_token}"
155
- )
159
+ f"{self.start_token}{reasoning_content}{self.end_token}")
160
final_output = model_output[end_index:]
161
162
if len(final_output) == 0:
vllm/reasoning/granite_reasoning_parser.py
renamed
+1
-2
@@ -8,9 +8,8 @@ from transformers import PreTrainedTokenizerBase
8
9
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
10
DeltaMessage)
11
-from vllm.entrypoints.openai.reasoning_parsers.abs_reasoning_parsers import (
12
- ReasoningParser, ReasoningParserManager)
11
from vllm.logger import init_logger
12
+from vllm.reasoning import ReasoningParser, ReasoningParserManager
13
14
logger = init_logger(__name__)
15