A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Bugfix]: Reasoning output bug according to the chat template change (#13025)
Signed-off-by: Ce Gao <cegao@tensorchord.ai>
Ce Gao committed
Feb 11, 2025 at 15:49 UTC
fc6485d27750076642e99a1ef2df0e6375958bb4
3 files changed
+129
-45
examples/online_serving/openai_chat_completion_with_reasoning.py
+4
-4
@@ -36,8 +36,8 @@ response = client.chat.completions.create(model=model, messages=messages)
36
reasoning_content = response.choices[0].message.reasoning_content
37
content = response.choices[0].message.content
38
39
-print("reasoning_content:", reasoning_content)
40
-print("content:", content)
39
+print("reasoning_content for Round 1:", reasoning_content)
40
+print("content for Round 1:", content)
41
42
# Round 2
43
messages.append({"role": "assistant", "content": content})
@@ -50,5 +50,5 @@ response = client.chat.completions.create(model=model, messages=messages)
50
reasoning_content = response.choices[0].message.reasoning_content
51
content = response.choices[0].message.content
52
53
-print("reasoning_content:", reasoning_content)
54
-print("content:", content)
53
+print("reasoning_content for Round 2:", reasoning_content)
54
+print("content for Round 2:", content)
tests/entrypoints/openai/reasoning_parsers/test_deepseekr1_reasoning_parser.py
+90
-18
@@ -15,32 +15,62 @@ start_token = "<think>"
15
end_token = "</think>"
16
17
SIMPLE_REASONING = {
18
- "output": "<think>This is a reasoning section</think>This is the rest",
18
+ "output": "This is a reasoning section</think>This is the rest",
19
"reasoning_content": "This is a reasoning section",
20
"content": "This is the rest",
21
}
22
COMPLETE_REASONING = {
23
- "output": "<think>This is a reasoning section</think>",
23
+ "output": "This is a reasoning section</think>",
24
"reasoning_content": "This is a reasoning section",
25
"content": None,
26
}
27
NO_REASONING = {
28
- "output": "This is a reasoning section",
28
+ "output": "This is content",
29
"reasoning_content": None,
30
- "content": "This is a reasoning section",
30
+ "content": "This is content",
31
+}
32
+NO_REASONING_STREAMING = {
33
+ "output": "This is a reasoning section",
34
+ "reasoning_content": "This is a reasoning section",
35
+ "content": None,
36
}
37
MULTIPLE_LINES = {
33
- "output": "<think>This\nThat</think>This is the rest\nThat",
38
+ "output": "This\nThat</think>This is the rest\nThat",
39
"reasoning_content": "This\nThat",
40
"content": "This is the rest\nThat",
41
}
42
SHORTEST_REASONING_NO_STREAMING = {
38
- "output": "<think></think>This is the rest",
43
+ "output": "</think>This is the rest",
44
"reasoning_content": "",
45
"content": "This is the rest",
46
}
47
SHORTEST_REASONING = {
43
- "output": "<think></think>This is the rest",
48
+ "output": "</think>This is the rest",
49
+ "reasoning_content": None,
50
+ "content": "This is the rest",
51
+}
52
+REASONING_WITH_THINK = {
53
+ "output": "<think>This is a reasoning section</think>This is the rest",
54
+ "reasoning_content": "This is a reasoning section",
55
+ "content": "This is the rest",
56
+}
57
+COMPLETE_REASONING_WITH_THINK = {
58
+ "output": "<think>This is a reasoning section</think>",
59
+ "reasoning_content": "This is a reasoning section",
60
+ "content": None,
61
+}
62
+MULTIPLE_LINES_WITH_THINK = {
63
+ "output": "<think>This\nThat</think>This is the rest\nThat",
64
+ "reasoning_content": "This\nThat",
65
+ "content": "This is the rest\nThat",
66
+}
67
+SHORTEST_REASONING_NO_STREAMING_WITH_THINK = {
68
+ "output": "</think>This is the rest",
69
+ "reasoning_content": "",
70
+ "content": "This is the rest",
71
+}
72
+SHORTEST_REASONING_WITH_THINK = {
73
+ "output": "</think>This is the rest",
74
"reasoning_content": None,
75
"content": "This is the rest",
76
}
@@ -49,37 +79,37 @@ TEST_CASES = [
79
pytest.param(
80
False,
81
SIMPLE_REASONING,
52
- id="simple_streaming",
82
+ id="simple_reasoning",
83
),
84
pytest.param(
85
True,
86
SIMPLE_REASONING,
57
- id="simple_streaming",
87
+ id="simple_reasoning_streaming",
88
),
89
pytest.param(
90
False,
91
COMPLETE_REASONING,
62
- id="complete_streaming",
92
+ id="complete_reasoning",
93
),
94
pytest.param(
95
True,
96
COMPLETE_REASONING,
67
- id="complete_streaming",
97
+ id="complete_reasoning_streaming",
98
),
99
pytest.param(
100
False,
101
NO_REASONING,
72
- id="no_streaming",
102
+ id="no_reasoning_token",
103
),
104
pytest.param(
105
True,
76
- NO_REASONING,
77
- id="no_streaming",
106
+ NO_REASONING_STREAMING,
107
+ id="no_reasoning_token_streaming",
108
),
109
pytest.param(
110
False,
111
MULTIPLE_LINES,
82
- id="multiple_lines_streaming",
112
+ id="multiple_lines",
113
),
114
pytest.param(
115
True,
@@ -89,23 +119,65 @@ TEST_CASES = [
119
pytest.param(
120
True,
121
SHORTEST_REASONING,
92
- id="shortest_streaming",
122
+ id="shortest",
123
),
124
pytest.param(
125
False,
126
SHORTEST_REASONING_NO_STREAMING,
127
id="shortest_streaming",
128
),
129
+ pytest.param(
130
+ False,
131
+ REASONING_WITH_THINK,
132
+ id="reasoning_with_think",
133
+ ),
134
+ pytest.param(
135
+ True,
136
+ REASONING_WITH_THINK,
137
+ id="reasoning_with_think_streaming",
138
+ ),
139
+ pytest.param(
140
+ False,
141
+ COMPLETE_REASONING_WITH_THINK,
142
+ id="complete_reasoning_with_think",
143
+ ),
144
+ pytest.param(
145
+ True,
146
+ COMPLETE_REASONING_WITH_THINK,
147
+ id="complete_reasoning_with_think_streaming",
148
+ ),
149
+ pytest.param(
150
+ False,
151
+ MULTIPLE_LINES_WITH_THINK,
152
+ id="multiple_lines_with_think",
153
+ ),
154
+ pytest.param(
155
+ True,
156
+ MULTIPLE_LINES_WITH_THINK,
157
+ id="multiple_lines_with_think_streaming",
158
+ ),
159
+ pytest.param(
160
+ False,
161
+ SHORTEST_REASONING_NO_STREAMING_WITH_THINK,
162
+ id="shortest_with_think",
163
+ ),
164
+ pytest.param(
165
+ True,
166
+ SHORTEST_REASONING_WITH_THINK,
167
+ id="shortest_with_think_streaming",
168
+ ),
169
]
170
171
+# Global tokenizer initialization to avoid repeated loading
172
+tokenizer = AutoTokenizer.from_pretrained("facebook/opt-125m")
173
+tokenizer.add_tokens([start_token, end_token])
174
+
175
176
@pytest.mark.parametrize("streaming, param_dict", TEST_CASES)
177
def test_reasoning(
178
streaming: bool,
179
param_dict: dict,
180
):
107
- tokenizer = AutoTokenizer.from_pretrained("facebook/opt-125m")
108
- tokenizer.add_tokens([start_token, end_token])
181
output = tokenizer.tokenize(param_dict["output"])
182
# decode everything to tokens
183
output_tokens: List[str] = [
vllm/entrypoints/openai/reasoning_parsers/deepseek_r1_reasoning_parser.py
+35
-23
@@ -67,6 +67,8 @@ class DeepSeekR1ReasoningParser(ReasoningParser):
67
]):
68
return None
69
70
+ # Check if <think> is present in previous or delta.
71
+ # Keep compatibility with models that don't generate <think> tokens.
72
if self.think_start_token_id in previous_token_ids:
73
if self.think_end_token_id in delta_token_ids:
74
# <think> in previous, </think> in delta,
@@ -85,7 +87,6 @@ class DeepSeekR1ReasoningParser(ReasoningParser):
87
# reasoning content continues
88
return DeltaMessage(reasoning_content=delta_text)
89
elif self.think_start_token_id in delta_token_ids:
88
- logger.info(delta_text)
90
if self.think_end_token_id in delta_token_ids:
91
# <think> in delta, </think> in delta, extract reasoning content
92
start_index = delta_text.find(self.think_start_token)
@@ -101,35 +102,46 @@ class DeepSeekR1ReasoningParser(ReasoningParser):
102
# reasoning content continues
103
return DeltaMessage(reasoning_content=delta_text)
104
else:
104
- # No <think> in previous or delta, reasoning content continues.
105
- return DeltaMessage(content=delta_text)
105
+ # No <think> in previous or delta, also need to check for </think>.
106
+ # Because the model may have generated </think> without <think>
107
+ # Ref https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f
108
+ if self.think_end_token_id in delta_token_ids:
109
+ # </think> in delta with more tokens,
110
+ # extract reasoning content and content
111
+ end_index = delta_text.find(self.think_end_token)
112
+ reasoning_content = delta_text[:end_index]
113
+ content = delta_text[end_index + len(self.think_end_token):]
114
+ return DeltaMessage(reasoning_content=reasoning_content,
115
+ content=content if content else None)
116
+ elif self.think_end_token_id in previous_token_ids:
117
+ # </think> in previous, thinking content ends
118
+ return DeltaMessage(content=delta_text)
119
+ else:
120
+ # no </think> in previous or delta, reasoning content continues
121
+ return DeltaMessage(reasoning_content=delta_text)
122
123
def extract_reasoning_content(
124
self, model_output: str, request: ChatCompletionRequest
125
) -> Tuple[Optional[str], Optional[str]]:
126
111
- # Check if the model output contains the <think> tokens.
112
- if (self.think_start_token not in model_output
113
- or self.think_end_token not in model_output):
127
+ # DeepSeek R1 doesn't generate <think> now.
128
+ # Thus we assume the reasoning content is always at the start.
129
+ # Ref https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f
130
+ if self.think_end_token not in model_output:
131
return None, model_output
132
else:
133
+ # Add a start token if it's missing to keep compatibility.
134
+ if self.think_start_token not in model_output:
135
+ model_output = f"{self.think_start_token}{model_output}"
136
# Use a regex to find the reasoning content
137
reasoning_content = self.reasoning_regex.findall(model_output)[0]
138
119
- # Remove the reasoning content from the model output
120
- # Although deepseek's <think> token is always at the
121
- # beginning of the line, we cannot guarantee that the
122
- # other models will follow this convention.
123
- # Therefore, we need to add :start_index.
124
- start_index = model_output.find(self.think_start_token)
125
- if start_index != -1:
126
- end_index = start_index + len(
127
- f"{self.think_start_token}{reasoning_content}{self.think_end_token}"
128
- )
129
- model_output = model_output[:start_index] + \
130
- model_output[end_index:]
131
-
132
- if len(model_output) == 0:
133
- return reasoning_content, None
134
-
135
- return reasoning_content, model_output
139
+ end_index = len(
140
+ f"{self.think_start_token}{reasoning_content}{self.think_end_token}"
141
+ )
142
+ final_output = model_output[end_index:]
143
+
144
+ if len(final_output) == 0:
145
+ return reasoning_content, None
146
+
147
+ return reasoning_content, final_output