A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Frontend] Support tool calling and reasoning parser (#14511)
Signed-off-by: WangErXiao <863579016@qq.com>
Robin committed
Mar 24, 2025 at 05:00 UTC
d6cd59f122351a71e742e57221df2f26b3b2f9f9
8 files changed
+555
-63
.buildkite/test-pipeline.yaml
+1
-1
@@ -118,7 +118,7 @@ steps:
118
- pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
119
- pytest -v -s entrypoints/llm/test_generate_multiple_loras.py # it needs a clean process
120
- VLLM_USE_V1=0 pytest -v -s entrypoints/llm/test_guided_generate.py # it needs a clean process
121
- - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/correctness/
121
+ - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/correctness/
122
- pytest -v -s entrypoints/test_chat_utils.py
123
- VLLM_USE_V1=0 pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
124
docs/source/features/reasoning_outputs.md
+46
-5
@@ -10,10 +10,10 @@ Reasoning models return a additional `reasoning_content` field in their outputs,
10
11
vLLM currently supports the following reasoning models:
12
13
-| Model Series | Parser Name | Structured Output Support |
14
-|--------------|-------------|------------------|
15
-| [DeepSeek R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d) | `deepseek_r1` | `guided_json`, `guided_regex` |
16
-| [QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) | `deepseek_r1` | `guided_json`, `guided_regex` |
13
+| Model Series | Parser Name | Structured Output Support | Tool Calling |
14
+|--------------|-------------|------------------|-------------|
15
+| [DeepSeek R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d) | `deepseek_r1` | `guided_json`, `guided_regex` | ❌ |
16
+| [QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) | `deepseek_r1` | `guided_json`, `guided_regex` | ✅ |
17
18
## Quickstart
19
@@ -170,10 +170,51 @@ print("reasoning_content: ", completion.choices[0].message.reasoning_content)
170
print("content: ", completion.choices[0].message.content)
171
```
172
173
+## Tool Calling
174
+
175
+The reasoning content is also available when both tool calling and the reasoning parser are enabled. Additionally, tool calling only parses functions from the `content` field, not from the `reasoning_content`.
176
+
177
+```python
178
+from openai import OpenAI
179
+
180
+client = OpenAI(base_url="http://localhost:8000/v1", api_key="dummy")
181
+
182
+tools = [{
183
+ "type": "function",
184
+ "function": {
185
+ "name": "get_weather",
186
+ "description": "Get the current weather in a given location",
187
+ "parameters": {
188
+ "type": "object",
189
+ "properties": {
190
+ "location": {"type": "string", "description": "City and state, e.g., 'San Francisco, CA'"},
191
+ "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}
192
+ },
193
+ "required": ["location", "unit"]
194
+ }
195
+ }
196
+}]
197
+
198
+response = client.chat.completions.create(
199
+ model=client.models.list().data[0].id,
200
+ messages=[{"role": "user", "content": "What's the weather like in San Francisco?"}],
201
+ tools=tools,
202
+ tool_choice="auto"
203
+)
204
+
205
+print(response)
206
+tool_call = response.choices[0].message.tool_calls[0].function
207
+
208
+print(f"reasoning_content: {response.choices[0].message.reasoning_content}")
209
+print(f"Function called: {tool_call.name}")
210
+print(f"Arguments: {tool_call.arguments}")
211
+```
212
+
213
+For more examples, please refer to <gh-file:examples/online_serving/openai_chat_completion_tool_calls_with_reasoning.py> .
214
+
215
## Limitations
216
217
- The reasoning content is only available for online serving's chat completion endpoint (`/v1/chat/completions`).
176
-- It is not compatible with [`tool_calling`](#tool_calling).
218
219
## How to support a new reasoning model
220
examples/online_serving/openai_chat_completion_tool_calls_with_reasoning.py
new
+177
@@ -0,0 +1,177 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+"""
3
+An example demonstrates how to use tool calling with reasoning models
4
+like QwQ-32B. The reasoning_content will not be parsed by the tool
5
+calling process; only the final output will be parsed.
6
+
7
+To run this example, you need to start the vLLM server with both
8
+the reasoning parser and tool calling enabled.
9
+
10
+```bash
11
+vllm serve Qwen/QwQ-32B \
12
+ --enable-reasoning --reasoning-parser deepseek_r1 \
13
+ --enable-auto-tool-choice --tool-call-parser hermes
14
+
15
+```
16
+
17
+"""
18
+
19
+from openai import OpenAI
20
+
21
+
22
+# Now, simulate a tool call
23
+def get_current_weather(city: str, state: str, unit: 'str'):
24
+ return ("The weather in Dallas, Texas is 85 degrees fahrenheit. It is "
25
+ "partly cloudly, with highs in the 90's.")
26
+
27
+
28
+available_tools = {"get_current_weather": get_current_weather}
29
+
30
+# Modify OpenAI's API key and API base to use vLLM's API server.
31
+openai_api_key = "EMPTY"
32
+openai_api_base = "http://localhost:8000/v1"
33
+
34
+client = OpenAI(
35
+ api_key=openai_api_key,
36
+ base_url=openai_api_base,
37
+)
38
+
39
+models = client.models.list()
40
+model = models.data[0].id
41
+
42
+tools = [{
43
+ "type": "function",
44
+ "function": {
45
+ "name": "get_current_weather",
46
+ "description": "Get the current weather in a given location",
47
+ "parameters": {
48
+ "type": "object",
49
+ "properties": {
50
+ "city": {
51
+ "type":
52
+ "string",
53
+ "description":
54
+ "The city to find the weather for, e.g. 'San Francisco'"
55
+ },
56
+ "state": {
57
+ "type":
58
+ "string",
59
+ "description":
60
+ "the two-letter abbreviation for the state that the city is"
61
+ " in, e.g. 'CA' which would mean 'California'"
62
+ },
63
+ "unit": {
64
+ "type": "string",
65
+ "description": "The unit to fetch the temperature in",
66
+ "enum": ["celsius", "fahrenheit"]
67
+ }
68
+ },
69
+ "required": ["city", "state", "unit"]
70
+ }
71
+ }
72
+}]
73
+messages = [{
74
+ "role": "user",
75
+ "content": "Hi! How are you doing today?"
76
+}, {
77
+ "role": "assistant",
78
+ "content": "I'm doing well! How can I help you?"
79
+}, {
80
+ "role":
81
+ "user",
82
+ "content":
83
+ "Can you tell me what the temperate will be in Dallas, in fahrenheit?"
84
+}]
85
+
86
+
87
+def extract_reasoning_and_calls(chunks: list):
88
+ reasoning_content = ""
89
+ tool_call_idx = -1
90
+ arguments = []
91
+ function_names = []
92
+ for chunk in chunks:
93
+ if chunk.choices[0].delta.tool_calls:
94
+ tool_call = chunk.choices[0].delta.tool_calls[0]
95
+ if tool_call.index != tool_call_idx:
96
+ tool_call_idx = chunk.choices[0].delta.tool_calls[0].index
97
+ arguments.append("")
98
+ function_names.append("")
99
+
100
+ if tool_call.function:
101
+ if tool_call.function.name:
102
+ function_names[tool_call_idx] = tool_call.function.name
103
+
104
+ if tool_call.function.arguments:
105
+ arguments[tool_call_idx] += tool_call.function.arguments
106
+ else:
107
+ if hasattr(chunk.choices[0].delta, "reasoning_content"):
108
+ reasoning_content += chunk.choices[0].delta.reasoning_content
109
+ return reasoning_content, arguments, function_names
110
+
111
+
112
+print("---------Full Generate With Automatic Function Calling-------------")
113
+tool_calls = client.chat.completions.create(messages=messages,
114
+ model=model,
115
+ tools=tools)
116
+print(f"reasoning_content: {tool_calls.choices[0].message.reasoning_content}")
117
+print(f"function name: "
118
+ f"{tool_calls.choices[0].message.tool_calls[0].function.name}")
119
+print(f"function arguments: "
120
+ f"{tool_calls.choices[0].message.tool_calls[0].function.arguments}")
121
+
122
+print("----------Stream Generate With Automatic Function Calling-----------")
123
+tool_calls_stream = client.chat.completions.create(messages=messages,
124
+ model=model,
125
+ tools=tools,
126
+ stream=True)
127
+chunks = []
128
+for chunk in tool_calls_stream:
129
+ chunks.append(chunk)
130
+
131
+reasoning_content, arguments, function_names = extract_reasoning_and_calls(
132
+ chunks)
133
+
134
+print(f"reasoning_content: {reasoning_content}")
135
+print(f"function name: {function_names[0]}")
136
+print(f"function arguments: {arguments[0]}")
137
+
138
+print("----------Full Generate With Named Function Calling-----------------")
139
+tool_calls = client.chat.completions.create(messages=messages,
140
+ model=model,
141
+ tools=tools,
142
+ tool_choice={
143
+ "type": "function",
144
+ "function": {
145
+ "name":
146
+ "get_current_weather"
147
+ }
148
+ })
149
+
150
+tool_call = tool_calls.choices[0].message.tool_calls[0].function
151
+print(f"reasoning_content: {tool_calls.choices[0].message.reasoning_content}")
152
+print(f"function name: {tool_call.name}")
153
+print(f"function arguments: {tool_call.arguments}")
154
+print("----------Stream Generate With Named Function Calling--------------")
155
+
156
+tool_calls_stream = client.chat.completions.create(
157
+ messages=messages,
158
+ model=model,
159
+ tools=tools,
160
+ tool_choice={
161
+ "type": "function",
162
+ "function": {
163
+ "name": "get_current_weather"
164
+ }
165
+ },
166
+ stream=True)
167
+
168
+chunks = []
169
+for chunk in tool_calls_stream:
170
+ chunks.append(chunk)
171
+
172
+reasoning_content, arguments, function_names = extract_reasoning_and_calls(
173
+ chunks)
174
+print(f"reasoning_content: {reasoning_content}")
175
+print(f"function name: {function_names[0]}")
176
+print(f"function arguments: {arguments[0]}")
177
+print("\n\n")
tests/entrypoints/openai/test_chat_with_tool_reasoning.py
new
+145
@@ -0,0 +1,145 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+import openai # use the official client for correctness check
4
+import pytest
5
+import pytest_asyncio
6
+
7
+from ...utils import RemoteOpenAIServer
8
+
9
+# a reasoning and tool calling model
10
+MODEL_NAME = "Qwen/QwQ-32B"
11
+
12
+
13
+@pytest.fixture(scope="module")
14
+def server(): # noqa: F811
15
+ args = [
16
+ "--max-model-len", "8192", "--enforce-eager", "--enable-reasoning",
17
+ "--reasoning-parser", "deepseek_r1", "--enable-auto-tool-choice",
18
+ "--tool-call-parser", "hermes"
19
+ ]
20
+
21
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
22
+ yield remote_server
23
+
24
+
25
+@pytest_asyncio.fixture
26
+async def client(server):
27
+ async with server.get_async_client() as async_client:
28
+ yield async_client
29
+
30
+
31
+TOOLS = [{
32
+ "type": "function",
33
+ "function": {
34
+ "name": "get_current_weather",
35
+ "description": "Get the current weather in a given location",
36
+ "parameters": {
37
+ "type": "object",
38
+ "properties": {
39
+ "city": {
40
+ "type":
41
+ "string",
42
+ "description":
43
+ "The city to find the weather for, e.g. 'San Francisco'"
44
+ },
45
+ "state": {
46
+ "type":
47
+ "string",
48
+ "description":
49
+ "the two-letter abbreviation for the state that the city is"
50
+ " in, e.g. 'CA' which would mean 'California'"
51
+ },
52
+ "unit": {
53
+ "type": "string",
54
+ "description": "The unit to fetch the temperature in",
55
+ "enum": ["celsius", "fahrenheit"]
56
+ }
57
+ },
58
+ "required": ["city", "state", "unit"]
59
+ }
60
+ }
61
+}]
62
+
63
+MESSAGES = [{
64
+ "role": "user",
65
+ "content": "Hi! How are you doing today?"
66
+}, {
67
+ "role": "assistant",
68
+ "content": "I'm doing well! How can I help you?"
69
+}, {
70
+ "role":
71
+ "user",
72
+ "content":
73
+ "Can you tell me what the temperate will be in Dallas, in fahrenheit?"
74
+}]
75
+
76
+FUNC_NAME = "get_current_weather"
77
+FUNC_ARGS = """{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}"""
78
+
79
+
80
+def extract_reasoning_and_calls(chunks: list):
81
+ reasoning_content = ""
82
+ tool_call_idx = -1
83
+ arguments = []
84
+ function_names = []
85
+ for chunk in chunks:
86
+ if chunk.choices[0].delta.tool_calls:
87
+ tool_call = chunk.choices[0].delta.tool_calls[0]
88
+ if tool_call.index != tool_call_idx:
89
+ tool_call_idx = chunk.choices[0].delta.tool_calls[0].index
90
+ arguments.append("")
91
+ function_names.append("")
92
+
93
+ if tool_call.function:
94
+ if tool_call.function.name:
95
+ function_names[tool_call_idx] = tool_call.function.name
96
+
97
+ if tool_call.function.arguments:
98
+ arguments[tool_call_idx] += tool_call.function.arguments
99
+ else:
100
+ if hasattr(chunk.choices[0].delta, "reasoning_content"):
101
+ reasoning_content += chunk.choices[0].delta.reasoning_content
102
+ return reasoning_content, arguments, function_names
103
+
104
+
105
+# test streaming
106
+@pytest.mark.asyncio
107
+async def test_chat_streaming_of_tool_and_reasoning(
108
+ client: openai.AsyncOpenAI):
109
+
110
+ stream = await client.chat.completions.create(
111
+ model=MODEL_NAME,
112
+ messages=MESSAGES,
113
+ tools=TOOLS,
114
+ temperature=0.0,
115
+ stream=True,
116
+ )
117
+
118
+ chunks = []
119
+ async for chunk in stream:
120
+ chunks.append(chunk)
121
+
122
+ reasoning_content, arguments, function_names = extract_reasoning_and_calls(
123
+ chunks)
124
+ assert len(reasoning_content) > 0
125
+ assert len(function_names) > 0 and function_names[0] == FUNC_NAME
126
+ assert len(arguments) > 0 and arguments[0] == FUNC_ARGS
127
+
128
+
129
+# test full generate
130
+@pytest.mark.asyncio
131
+async def test_chat_full_of_tool_and_reasoning(client: openai.AsyncOpenAI):
132
+
133
+ tool_calls = await client.chat.completions.create(
134
+ model=MODEL_NAME,
135
+ messages=MESSAGES,
136
+ tools=TOOLS,
137
+ temperature=0.0,
138
+ stream=False,
139
+ )
140
+
141
+ assert len(tool_calls.choices[0].message.reasoning_content) > 0
142
+ assert tool_calls.choices[0].message.tool_calls[0].function.name \
143
+ == FUNC_NAME
144
+ assert tool_calls.choices[0].message.tool_calls[0].function.arguments \
145
+ == FUNC_ARGS
vllm/entrypoints/openai/cli_args.py
-7
@@ -289,13 +289,6 @@ def validate_parsed_serve_args(args: argparse.Namespace):
289
raise TypeError("Error: --enable-reasoning requires "
290
"--reasoning-parser")
291
292
- # Ref https://api-docs.deepseek.com/guides/reasoning_model
293
- # tool call and reasoning cannot be enabled at the same time.
294
- if args.enable_auto_tool_choice and args.enable_reasoning:
295
- raise TypeError(
296
- "Error: --enable-auto-tool-choice and "
297
- "--enable-reasoning cannot be enabled at the same time")
298
-
292
293
def create_parser_for_docs() -> FlexibleArgumentParser:
294
parser_for_docs = FlexibleArgumentParser(
vllm/entrypoints/openai/reasoning_parsers/abs_reasoning_parsers.py
+35
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import os
4
+from abc import abstractmethod
5
from collections.abc import Sequence
6
from functools import cached_property
7
from typing import Callable, Optional, Union
@@ -76,6 +77,40 @@ class ReasoningParser:
77
"AbstractReasoningParser.extract_reasoning_content_streaming "
78
"has not been implemented!")
79
80
+ # TODO: need to rebase by PR #14428
81
+ @abstractmethod
82
+ def is_reasoning_end(self, input_ids: list[int]) -> bool:
83
+ """
84
+ Check if the reasoning content ends in the input_ids.
85
+ Parameters:
86
+ input_ids: list[int]
87
+ The input_ids of the model output.
88
+ Returns:
89
+ bool
90
+ True if the reasoning content ends in the input_ids.
91
+ """
92
+
93
+ raise NotImplementedError(
94
+ "AbstractReasoningParser.is_reasoning_end has"
95
+ "not been implemented!")
96
+
97
+ # TODO: need to rebase by PR #14428
98
+ @abstractmethod
99
+ def extract_content_ids(self, input_ids: list[int]) -> list[int]:
100
+ """
101
+ Extract content token ids from the input_ids.
102
+ Parameters:
103
+ input_ids: list[int]
104
+ The input_ids of the model output.
105
+ Returns:
106
+ list[int]
107
+ The extracted content from the input_ids.
108
+ """
109
+
110
+ raise NotImplementedError(
111
+ "AbstractReasoningParser.extract_content_ids has"
112
+ " not been implemented!")
113
+
114
115
class ReasoningParserManager:
116
reasoning_parsers: dict[str, type] = {}
vllm/entrypoints/openai/reasoning_parsers/deepseek_r1_reasoning_parser.py
+13
@@ -45,6 +45,19 @@ class DeepSeekR1ReasoningParser(ReasoningParser):
45
"DeepSeek R1 reasoning parser could not locate think start/end "
46
"tokens in the tokenizer!")
47
48
+ # TODO: need to rebase by PR #14428
49
+ def is_reasoning_end(self, input_ids: list[int]) -> bool:
50
+ return self.think_end_token_id in input_ids
51
+
52
+ def extract_content_ids(self, input_ids: list[int]) -> list[int]:
53
+ """
54
+ Extract the content after the end tokens
55
+ """
56
+ if self.think_end_token_id not in input_ids[:-1]:
57
+ return []
58
+ else:
59
+ return input_ids[input_ids.index(self.think_end_token_id) + 1:]
60
+
61
def extract_reasoning_content_streaming(
62
self,
63
previous_text: str,
vllm/entrypoints/openai/serving_chat.py
+138
-50
@@ -328,6 +328,9 @@ class OpenAIServingChat(OpenAIServing):
328
# These are only required in "auto" tool choice case
329
previous_texts = [""] * num_choices
330
all_previous_token_ids = [[]] * num_choices
331
+ # For reasoning parser and tool call all enabled
332
+ added_content_delta_arr = [False] * num_choices
333
+ reasoning_end_arr = [False] * num_choices
334
else:
335
previous_texts, all_previous_token_ids = None, None
336
@@ -477,27 +480,116 @@ class OpenAIServingChat(OpenAIServing):
480
481
delta_message: Optional[DeltaMessage]
482
480
- # handle streaming deltas for tools with named tool_choice
481
- if tool_choice_function_name:
482
- delta_message = DeltaMessage(tool_calls=[
483
- DeltaToolCall(function=DeltaFunctionCall(
484
- name=tool_choice_function_name,
485
- arguments=delta_text),
486
- index=i)
487
- ])
488
-
489
- # handle streaming deltas for tools with "auto" tool choice
490
- elif tool_choice_auto:
483
+ # just update previous_texts and previous_token_ids
484
+ if tool_choice_auto or should_stream_with_reasoning_parsing:
485
assert previous_texts is not None
486
assert all_previous_token_ids is not None
493
- assert tool_parser is not None
494
- #TODO optimize manipulation of these lists
487
previous_text = previous_texts[i]
488
previous_token_ids = all_previous_token_ids[i]
489
current_text = previous_text + delta_text
490
current_token_ids = previous_token_ids + list(
491
output.token_ids)
492
493
+ # handle streaming deltas for tools with named tool_choice
494
+ if tool_choice_function_name:
495
+ if (self.enable_reasoning
496
+ and not reasoning_parser.is_reasoning_end(
497
+ previous_token_ids)):
498
+ assert reasoning_parser is not None
499
+ delta_message = (
500
+ reasoning_parser.
501
+ extract_reasoning_content_streaming(
502
+ previous_text,
503
+ current_text,
504
+ delta_text,
505
+ previous_token_ids,
506
+ current_token_ids,
507
+ output.token_ids,
508
+ ))
509
+ # When encountering think end id in delta_token_ids,
510
+ # process the `content`. Only keep 'content',
511
+ # remove 'reasoning_content'
512
+ if reasoning_parser.is_reasoning_end(
513
+ list(output.token_ids)):
514
+ if delta_message and delta_message.content:
515
+ # This need to be added to next `delta_text`
516
+ current_text = delta_message.content
517
+ delta_message.content = None
518
+ else:
519
+ current_text = ""
520
+ else:
521
+ # Just to add remaining `content`
522
+ if self.enable_reasoning:
523
+ delta_text = previous_text + delta_text
524
+ current_text = ""
525
+
526
+ delta_message = DeltaMessage(tool_calls=[
527
+ DeltaToolCall(function=DeltaFunctionCall(
528
+ name=tool_choice_function_name,
529
+ arguments=delta_text),
530
+ index=i)
531
+ ])
532
+
533
+ # handle streaming deltas for tools with "auto" tool choice
534
+ # and reasoning parser
535
+ elif tool_choice_auto and self.enable_reasoning:
536
+ assert tool_parser is not None
537
+ assert reasoning_parser is not None
538
+ assert added_content_delta_arr is not None
539
+ assert reasoning_end_arr is not None
540
+ if not reasoning_end_arr[i]:
541
+ delta_message = (
542
+ reasoning_parser.
543
+ extract_reasoning_content_streaming(
544
+ previous_text,
545
+ current_text,
546
+ delta_text,
547
+ previous_token_ids,
548
+ current_token_ids,
549
+ output.token_ids,
550
+ ))
551
+
552
+ # When encountering think end id in delta_token_ids,
553
+ # set reasoning status to end.
554
+ # Remove the text and token ids related
555
+ # to 'reasoning_content'.
556
+ if reasoning_parser.is_reasoning_end(
557
+ list(output.token_ids)):
558
+ reasoning_end_arr[i] = True
559
+ current_token_ids = \
560
+ reasoning_parser.extract_content_ids(
561
+ list(output.token_ids))
562
+ if delta_message and delta_message.content:
563
+ current_text = delta_message.content
564
+ delta_message.content = None
565
+ else:
566
+ current_text = ""
567
+
568
+ # handle tool calls only after reasoning is done,
569
+ else:
570
+ delta_token_ids = list(output.token_ids)
571
+ # First time to tool call,
572
+ # add the remaining text and token ids
573
+ # to delta from previous
574
+ if not added_content_delta_arr[i]:
575
+ added_content_delta_arr[i] = True
576
+ previous_text = ""
577
+ previous_token_ids = []
578
+ delta_text = current_text
579
+ delta_token_ids = current_token_ids
580
+
581
+ delta_message = (
582
+ tool_parser.extract_tool_calls_streaming(
583
+ previous_text=previous_text,
584
+ current_text=current_text,
585
+ delta_text=delta_text,
586
+ previous_token_ids=previous_token_ids,
587
+ current_token_ids=current_token_ids,
588
+ delta_token_ids=delta_token_ids,
589
+ request=request))
590
+ # when only tool calls
591
+ elif tool_choice_auto:
592
+ assert tool_parser is not None
593
delta_message = (
594
tool_parser.extract_tool_calls_streaming(
595
previous_text=previous_text,
@@ -507,23 +599,9 @@ class OpenAIServingChat(OpenAIServing):
599
current_token_ids=current_token_ids,
600
delta_token_ids=output.token_ids,
601
request=request))
510
-
511
- # update the previous values for the next iteration
512
- previous_texts[i] = current_text
513
- all_previous_token_ids[i] = current_token_ids
514
- # reasoning_content cannot be enabled with tool_choice.
515
- # If it is, the tool_choice will be used instead.
602
+ # when only reasoning
603
elif self.enable_reasoning:
517
- # handle reasoning_content delta
604
assert reasoning_parser is not None
519
- assert previous_texts is not None
520
- assert all_previous_token_ids is not None
521
- previous_text = previous_texts[i]
522
- previous_token_ids = all_previous_token_ids[i]
523
- current_text = previous_text + delta_text
524
- current_token_ids = previous_token_ids + list(
525
- output.token_ids)
526
-
605
delta_message = (reasoning_parser.
606
extract_reasoning_content_streaming(
607
previous_text,
@@ -533,15 +611,17 @@ class OpenAIServingChat(OpenAIServing):
611
current_token_ids,
612
output.token_ids,
613
))
536
-
537
- # update the previous values for the next iteration
538
- previous_texts[i] = current_text
539
- all_previous_token_ids[i] = current_token_ids
540
-
614
# handle streaming just a content delta
615
else:
616
delta_message = DeltaMessage(content=delta_text)
617
618
+ # update the previous values for the next iteration
619
+ if tool_choice_auto or should_stream_with_reasoning_parsing:
620
+ assert previous_texts is not None
621
+ assert all_previous_token_ids is not None
622
+ previous_texts[i] = current_text
623
+ all_previous_token_ids[i] = current_token_ids
624
+
625
# set the previous values for the next iteration
626
previous_num_tokens[i] += len(output.token_ids)
627
@@ -739,24 +819,24 @@ class OpenAIServingChat(OpenAIServing):
819
except RuntimeError as e:
820
logger.exception("Error in reasoning parser creation.")
821
return self.create_error_response(str(e))
742
-
822
+ # If the reasoning parser is enabled,
823
+ # tool calls are extracted exclusively from the content.
824
reasoning_content, content = (
825
reasoning_parser.extract_reasoning_content(
826
output.text, request=request))
746
-
747
- if reasoning_content:
748
- message = ChatMessage(role=role,
749
- content=content,
750
- reasoning_content=reasoning_content)
751
- else:
752
- message = ChatMessage(role=role, content=output.text)
827
+ else:
828
+ reasoning_content = None
829
+ content = output.text
830
831
# if auto tools are not enabled, and a named tool choice using
832
# outlines is not being used
756
- elif (not self.enable_auto_tools
757
- or not self.tool_parser) and not isinstance(
758
- request.tool_choice, ChatCompletionNamedToolChoiceParam):
759
- message = ChatMessage(role=role, content=output.text)
833
+ if (not self.enable_auto_tools
834
+ or not self.tool_parser) and not isinstance(
835
+ request.tool_choice,
836
+ ChatCompletionNamedToolChoiceParam):
837
+ message = ChatMessage(role=role,
838
+ reasoning_content=reasoning_content,
839
+ content=content)
840
841
# if the request uses tools and specified a tool choice
842
elif request.tool_choice and type(
@@ -766,18 +846,21 @@ class OpenAIServingChat(OpenAIServing):
846
tokenizer, MistralTokenizer) else ToolCall
847
message = ChatMessage(
848
role=role,
849
+ reasoning_content=reasoning_content,
850
content="",
851
tool_calls=[
852
tool_call_class(function=FunctionCall(
853
name=request.tool_choice.function.name,
773
- arguments=output.text))
854
+ arguments=content))
855
])
856
857
# if the request doesn't use tool choice
858
# OR specifies to not use a tool
859
elif not request.tool_choice or request.tool_choice == "none":
860
780
- message = ChatMessage(role=role, content=output.text)
861
+ message = ChatMessage(role=role,
862
+ reasoning_content=reasoning_content,
863
+ content=content)
864
865
# handle when there are tools and tool choice is auto
866
elif request.tools and (
@@ -792,20 +875,23 @@ class OpenAIServingChat(OpenAIServing):
875
return self.create_error_response(str(e))
876
877
tool_call_info = tool_parser.extract_tool_calls(
795
- output.text, request=request)
878
+ content if content is not None else "", request=request)
879
# In the OpenAI API the finish_reason is "tools_called"
880
# if the tool choice is auto and the model produced a tool
881
# call. The same is not true for named function calls
882
auto_tools_called = tool_call_info.tools_called
883
if tool_call_info.tools_called:
884
message = ChatMessage(role=role,
885
+ reasoning_content=reasoning_content,
886
content=tool_call_info.content,
887
tool_calls=tool_call_info.tool_calls)
888
889
else:
890
# FOR NOW make it a chat message; we will have to detect
891
# the type to make it later.
808
- message = ChatMessage(role=role, content=output.text)
892
+ message = ChatMessage(role=role,
893
+ reasoning_content=reasoning_content,
894
+ content=content)
895
896
# undetermined case that is still important to handle
897
else:
@@ -813,7 +899,9 @@ class OpenAIServingChat(OpenAIServing):
899
"Error in chat_completion_full_generator - cannot determine"
900
" if tools should be extracted. Returning a standard chat "
901
"completion.")
816
- message = ChatMessage(role=role, content=output.text)
902
+ message = ChatMessage(role=role,
903
+ reasoning_content=reasoning_content,
904
+ content=content)
905
906
choice_data = ChatCompletionResponseChoice(
907
index=output.index,