@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fc6485d277
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Bugfix]: Reasoning output bug according to the chat template change (#13025)

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

Ce Gao committed Feb 11, 2025 at 15:49 UTC fc6485d27750076642e99a1ef2df0e6375958bb4
3 files changed +129 -45
examples/online_serving/openai_chat_completion_with_reasoning.py
+4 -4
@@ -36,8 +36,8 @@ response = client.chat.completions.create(model=model, messages=messages)
36 reasoning_content = response.choices[0].message.reasoning_content
37 content = response.choices[0].message.content
38
39 -print("reasoning_content:", reasoning_content)
40 -print("content:", content)
39 +print("reasoning_content for Round 1:", reasoning_content)
40 +print("content for Round 1:", content)
41
42 # Round 2
43 messages.append({"role": "assistant", "content": content})
@@ -50,5 +50,5 @@ response = client.chat.completions.create(model=model, messages=messages)
50 reasoning_content = response.choices[0].message.reasoning_content
51 content = response.choices[0].message.content
52
53 -print("reasoning_content:", reasoning_content)
54 -print("content:", content)
53 +print("reasoning_content for Round 2:", reasoning_content)
54 +print("content for Round 2:", content)
tests/entrypoints/openai/reasoning_parsers/test_deepseekr1_reasoning_parser.py
+90 -18
@@ -15,32 +15,62 @@ start_token = "<think>"
15 end_token = "</think>"
16
17 SIMPLE_REASONING = {
18 - "output": "<think>This is a reasoning section</think>This is the rest",
18 + "output": "This is a reasoning section</think>This is the rest",
19 "reasoning_content": "This is a reasoning section",
20 "content": "This is the rest",
21 }
22 COMPLETE_REASONING = {
23 - "output": "<think>This is a reasoning section</think>",
23 + "output": "This is a reasoning section</think>",
24 "reasoning_content": "This is a reasoning section",
25 "content": None,
26 }
27 NO_REASONING = {
28 - "output": "This is a reasoning section",
28 + "output": "This is content",
29 "reasoning_content": None,
30 - "content": "This is a reasoning section",
30 + "content": "This is content",
31 +}
32 +NO_REASONING_STREAMING = {
33 + "output": "This is a reasoning section",
34 + "reasoning_content": "This is a reasoning section",
35 + "content": None,
36 }
37 MULTIPLE_LINES = {
33 - "output": "<think>This\nThat</think>This is the rest\nThat",
38 + "output": "This\nThat</think>This is the rest\nThat",
39 "reasoning_content": "This\nThat",
40 "content": "This is the rest\nThat",
41 }
42 SHORTEST_REASONING_NO_STREAMING = {
38 - "output": "<think></think>This is the rest",
43 + "output": "</think>This is the rest",
44 "reasoning_content": "",
45 "content": "This is the rest",
46 }
47 SHORTEST_REASONING = {
43 - "output": "<think></think>This is the rest",
48 + "output": "</think>This is the rest",
49 + "reasoning_content": None,
50 + "content": "This is the rest",
51 +}
52 +REASONING_WITH_THINK = {
53 + "output": "<think>This is a reasoning section</think>This is the rest",
54 + "reasoning_content": "This is a reasoning section",
55 + "content": "This is the rest",
56 +}
57 +COMPLETE_REASONING_WITH_THINK = {
58 + "output": "<think>This is a reasoning section</think>",
59 + "reasoning_content": "This is a reasoning section",
60 + "content": None,
61 +}
62 +MULTIPLE_LINES_WITH_THINK = {
63 + "output": "<think>This\nThat</think>This is the rest\nThat",
64 + "reasoning_content": "This\nThat",
65 + "content": "This is the rest\nThat",
66 +}
67 +SHORTEST_REASONING_NO_STREAMING_WITH_THINK = {
68 + "output": "</think>This is the rest",
69 + "reasoning_content": "",
70 + "content": "This is the rest",
71 +}
72 +SHORTEST_REASONING_WITH_THINK = {
73 + "output": "</think>This is the rest",
74 "reasoning_content": None,
75 "content": "This is the rest",
76 }
@@ -49,37 +79,37 @@ TEST_CASES = [
79 pytest.param(
80 False,
81 SIMPLE_REASONING,
52 - id="simple_streaming",
82 + id="simple_reasoning",
83 ),
84 pytest.param(
85 True,
86 SIMPLE_REASONING,
57 - id="simple_streaming",
87 + id="simple_reasoning_streaming",
88 ),
89 pytest.param(
90 False,
91 COMPLETE_REASONING,
62 - id="complete_streaming",
92 + id="complete_reasoning",
93 ),
94 pytest.param(
95 True,
96 COMPLETE_REASONING,
67 - id="complete_streaming",
97 + id="complete_reasoning_streaming",
98 ),
99 pytest.param(
100 False,
101 NO_REASONING,
72 - id="no_streaming",
102 + id="no_reasoning_token",
103 ),
104 pytest.param(
105 True,
76 - NO_REASONING,
77 - id="no_streaming",
106 + NO_REASONING_STREAMING,
107 + id="no_reasoning_token_streaming",
108 ),
109 pytest.param(
110 False,
111 MULTIPLE_LINES,
82 - id="multiple_lines_streaming",
112 + id="multiple_lines",
113 ),
114 pytest.param(
115 True,
@@ -89,23 +119,65 @@ TEST_CASES = [
119 pytest.param(
120 True,
121 SHORTEST_REASONING,
92 - id="shortest_streaming",
122 + id="shortest",
123 ),
124 pytest.param(
125 False,
126 SHORTEST_REASONING_NO_STREAMING,
127 id="shortest_streaming",
128 ),
129 + pytest.param(
130 + False,
131 + REASONING_WITH_THINK,
132 + id="reasoning_with_think",
133 + ),
134 + pytest.param(
135 + True,
136 + REASONING_WITH_THINK,
137 + id="reasoning_with_think_streaming",
138 + ),
139 + pytest.param(
140 + False,
141 + COMPLETE_REASONING_WITH_THINK,
142 + id="complete_reasoning_with_think",
143 + ),
144 + pytest.param(
145 + True,
146 + COMPLETE_REASONING_WITH_THINK,
147 + id="complete_reasoning_with_think_streaming",
148 + ),
149 + pytest.param(
150 + False,
151 + MULTIPLE_LINES_WITH_THINK,
152 + id="multiple_lines_with_think",
153 + ),
154 + pytest.param(
155 + True,
156 + MULTIPLE_LINES_WITH_THINK,
157 + id="multiple_lines_with_think_streaming",
158 + ),
159 + pytest.param(
160 + False,
161 + SHORTEST_REASONING_NO_STREAMING_WITH_THINK,
162 + id="shortest_with_think",
163 + ),
164 + pytest.param(
165 + True,
166 + SHORTEST_REASONING_WITH_THINK,
167 + id="shortest_with_think_streaming",
168 + ),
169 ]
170
171 +# Global tokenizer initialization to avoid repeated loading
172 +tokenizer = AutoTokenizer.from_pretrained("facebook/opt-125m")
173 +tokenizer.add_tokens([start_token, end_token])
174 +
175
176 @pytest.mark.parametrize("streaming, param_dict", TEST_CASES)
177 def test_reasoning(
178 streaming: bool,
179 param_dict: dict,
180 ):
107 - tokenizer = AutoTokenizer.from_pretrained("facebook/opt-125m")
108 - tokenizer.add_tokens([start_token, end_token])
181 output = tokenizer.tokenize(param_dict["output"])
182 # decode everything to tokens
183 output_tokens: List[str] = [
vllm/entrypoints/openai/reasoning_parsers/deepseek_r1_reasoning_parser.py
+35 -23
@@ -67,6 +67,8 @@ class DeepSeekR1ReasoningParser(ReasoningParser):
67 ]):
68 return None
69
70 + # Check if <think> is present in previous or delta.
71 + # Keep compatibility with models that don't generate <think> tokens.
72 if self.think_start_token_id in previous_token_ids:
73 if self.think_end_token_id in delta_token_ids:
74 # <think> in previous, </think> in delta,
@@ -85,7 +87,6 @@ class DeepSeekR1ReasoningParser(ReasoningParser):
87 # reasoning content continues
88 return DeltaMessage(reasoning_content=delta_text)
89 elif self.think_start_token_id in delta_token_ids:
88 - logger.info(delta_text)
90 if self.think_end_token_id in delta_token_ids:
91 # <think> in delta, </think> in delta, extract reasoning content
92 start_index = delta_text.find(self.think_start_token)
@@ -101,35 +102,46 @@ class DeepSeekR1ReasoningParser(ReasoningParser):
102 # reasoning content continues
103 return DeltaMessage(reasoning_content=delta_text)
104 else:
104 - # No <think> in previous or delta, reasoning content continues.
105 - return DeltaMessage(content=delta_text)
105 + # No <think> in previous or delta, also need to check for </think>.
106 + # Because the model may have generated </think> without <think>
107 + # Ref https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f
108 + if self.think_end_token_id in delta_token_ids:
109 + # </think> in delta with more tokens,
110 + # extract reasoning content and content
111 + end_index = delta_text.find(self.think_end_token)
112 + reasoning_content = delta_text[:end_index]
113 + content = delta_text[end_index + len(self.think_end_token):]
114 + return DeltaMessage(reasoning_content=reasoning_content,
115 + content=content if content else None)
116 + elif self.think_end_token_id in previous_token_ids:
117 + # </think> in previous, thinking content ends
118 + return DeltaMessage(content=delta_text)
119 + else:
120 + # no </think> in previous or delta, reasoning content continues
121 + return DeltaMessage(reasoning_content=delta_text)
122
123 def extract_reasoning_content(
124 self, model_output: str, request: ChatCompletionRequest
125 ) -> Tuple[Optional[str], Optional[str]]:
126
111 - # Check if the model output contains the <think> tokens.
112 - if (self.think_start_token not in model_output
113 - or self.think_end_token not in model_output):
127 + # DeepSeek R1 doesn't generate <think> now.
128 + # Thus we assume the reasoning content is always at the start.
129 + # Ref https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f
130 + if self.think_end_token not in model_output:
131 return None, model_output
132 else:
133 + # Add a start token if it's missing to keep compatibility.
134 + if self.think_start_token not in model_output:
135 + model_output = f"{self.think_start_token}{model_output}"
136 # Use a regex to find the reasoning content
137 reasoning_content = self.reasoning_regex.findall(model_output)[0]
138
119 - # Remove the reasoning content from the model output
120 - # Although deepseek's <think> token is always at the
121 - # beginning of the line, we cannot guarantee that the
122 - # other models will follow this convention.
123 - # Therefore, we need to add :start_index.
124 - start_index = model_output.find(self.think_start_token)
125 - if start_index != -1:
126 - end_index = start_index + len(
127 - f"{self.think_start_token}{reasoning_content}{self.think_end_token}"
128 - )
129 - model_output = model_output[:start_index] + \
130 - model_output[end_index:]
131 -
132 - if len(model_output) == 0:
133 - return reasoning_content, None
134 -
135 - return reasoning_content, model_output
139 + end_index = len(
140 + f"{self.think_start_token}{reasoning_content}{self.think_end_token}"
141 + )
142 + final_output = model_output[end_index:]
143 +
144 + if len(final_output) == 0:
145 + return reasoning_content, None
146 +
147 + return reasoning_content, final_output