A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Frontend][Feature] Add jamba tool parser (#9154)
tomeras91 committed
Oct 18, 2024 at 13:27 UTC
d2b1bf55ec0d50f76762b902ca84036ac53e9646
6 files changed
+595
-9
docs/source/serving/openai_compatible_server.md
+14
-6
@@ -157,7 +157,7 @@ vLLM will use guided decoding to ensure the response matches the tool parameter
157
To enable this feature, you should set the following flags:
158
* `--enable-auto-tool-choice` -- **mandatory** Auto tool choice. tells vLLM that you want to enable the model to generate its own tool calls when it
159
deems appropriate.
160
-* `--tool-call-parser` -- select the tool parser to use - currently either `hermes` or `mistral` or `llama3_json` or `internlm`. Additional tool parsers
160
+* `--tool-call-parser` -- select the tool parser to use (listed below). Additional tool parsers
161
will continue to be added in the future, and also can register your own tool parsers in the `--tool-parser-plugin`.
162
* `--tool-parser-plugin` -- **optional** tool parser plugin used to register user defined tool parsers into vllm, the registered tool parser name can be specified in `--tool-call-parser`.
163
* `--chat-template` -- **optional** for auto tool choice. the path to the chat template which handles `tool`-role messages and `assistant`-role messages
@@ -168,7 +168,7 @@ from HuggingFace; and you can find an example of this in a `tokenizer_config.jso
168
169
If your favorite tool-calling model is not supported, please feel free to contribute a parser & tool use chat template!
170
171
-#### Hermes Models
171
+#### Hermes Models (`hermes`)
172
All Nous Research Hermes-series models newer than Hermes 2 Pro should be supported.
173
* `NousResearch/Hermes-2-Pro-*`
174
* `NousResearch/Hermes-2-Theta-*`
@@ -180,7 +180,7 @@ step in their creation_.
180
181
Flags: `--tool-call-parser hermes`
182
183
-#### Mistral Models
183
+#### Mistral Models (`mistral`)
184
Supported models:
185
* `mistralai/Mistral-7B-Instruct-v0.3` (confirmed)
186
* Additional mistral function-calling models are compatible as well.
@@ -199,7 +199,7 @@ when tools are provided, that results in much better reliability when working wi
199
200
Recommended flags: `--tool-call-parser mistral --chat-template examples/tool_chat_template_mistral_parallel.jinja`
201
202
-#### Llama Models
202
+#### Llama Models (`llama3_json`)
203
Supported models:
204
* `meta-llama/Meta-Llama-3.1-8B-Instruct`
205
* `meta-llama/Meta-Llama-3.1-70B-Instruct`
@@ -219,16 +219,24 @@ it works better with vLLM.
219
220
Recommended flags: `--tool-call-parser llama3_json --chat-template examples/tool_chat_template_llama3_json.jinja`
221
222
-#### Internlm Models
222
+#### InternLM Models (`internlm`)
223
Supported models:
224
* `internlm/internlm2_5-7b-chat` (confirmed)
225
* Additional internlm2.5 function-calling models are compatible as well
226
227
Known issues:
228
-* Although this implementation also supports Internlm2, the tool call results are not stable when testing with the `internlm/internlm2-chat-7b` model.
228
+* Although this implementation also supports InternLM2, the tool call results are not stable when testing with the `internlm/internlm2-chat-7b` model.
229
230
Recommended flags: `--tool-call-parser internlm --chat-template examples/tool_chat_template_internlm2_tool.jinja`
231
232
+#### Jamba Models (`jamba`)
233
+AI21's Jamba-1.5 models are supported.
234
+* `ai21labs/AI21-Jamba-1.5-Mini`
235
+* `ai21labs/AI21-Jamba-1.5-Large`
236
+
237
+
238
+Flags: `--tool-call-parser jamba`
239
+
240
241
### How to write a tool parser plugin
242
tests/tool_use/test_jamba_tool_parser.py
new
+275
@@ -0,0 +1,275 @@
1
+import json
2
+from typing import Generator, List, Optional
3
+
4
+import partial_json_parser
5
+import pytest
6
+from partial_json_parser.core.options import Allow
7
+
8
+from vllm.entrypoints.openai.protocol import (DeltaMessage, FunctionCall,
9
+ ToolCall)
10
+from vllm.entrypoints.openai.tool_parsers import JambaToolParser
11
+from vllm.transformers_utils.detokenizer import detokenize_incrementally
12
+from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
13
+
14
+MODEL = "ai21labs/Jamba-tiny-dev"
15
+
16
+
17
+@pytest.fixture(scope="module")
18
+def jamba_tokenizer():
19
+ return get_tokenizer(tokenizer_name=MODEL)
20
+
21
+
22
+@pytest.fixture
23
+def jamba_tool_parser(jamba_tokenizer):
24
+ return JambaToolParser(jamba_tokenizer)
25
+
26
+
27
+def assert_tool_calls(actual_tool_calls: List[ToolCall],
28
+ expected_tool_calls: List[ToolCall]):
29
+ assert len(actual_tool_calls) == len(expected_tool_calls)
30
+
31
+ for actual_tool_call, expected_tool_call in zip(actual_tool_calls,
32
+ expected_tool_calls):
33
+ assert isinstance(actual_tool_call.id, str)
34
+ assert len(actual_tool_call.id) > 16
35
+
36
+ assert actual_tool_call.type == "function"
37
+ assert actual_tool_call.function == expected_tool_call.function
38
+
39
+
40
+def stream_delta_message_generator(
41
+ jamba_tool_parser: JambaToolParser, jamba_tokenizer: AnyTokenizer,
42
+ model_output: str) -> Generator[DeltaMessage, None, None]:
43
+ all_token_ids = jamba_tokenizer.encode(model_output,
44
+ add_special_tokens=False)
45
+
46
+ previous_text = ""
47
+ previous_tokens = None
48
+ prefix_offset = 0
49
+ read_offset = 0
50
+ for i, delta_token in enumerate(all_token_ids):
51
+ delta_token_ids = [delta_token]
52
+ previous_token_ids = all_token_ids[:i]
53
+ current_token_ids = all_token_ids[:i + 1]
54
+
55
+ (new_tokens, delta_text, new_prefix_offset,
56
+ new_read_offset) = detokenize_incrementally(
57
+ tokenizer=jamba_tokenizer,
58
+ all_input_ids=current_token_ids,
59
+ prev_tokens=previous_tokens,
60
+ prefix_offset=prefix_offset,
61
+ read_offset=read_offset,
62
+ skip_special_tokens=False,
63
+ spaces_between_special_tokens=True,
64
+ )
65
+
66
+ current_text = previous_text + delta_text
67
+
68
+ delta_message = jamba_tool_parser.extract_tool_calls_streaming(
69
+ previous_text,
70
+ current_text,
71
+ delta_text,
72
+ previous_token_ids,
73
+ current_token_ids,
74
+ delta_token_ids,
75
+ request=None, # type: ignore[arg-type]
76
+ )
77
+ if delta_message:
78
+ yield delta_message
79
+
80
+ previous_text = current_text
81
+ previous_tokens = previous_tokens + new_tokens if previous_tokens\
82
+ else new_tokens
83
+ prefix_offset = new_prefix_offset
84
+ read_offset = new_read_offset
85
+
86
+
87
+def test_extract_tool_calls_no_tools(jamba_tool_parser):
88
+ model_output = "This is a test"
89
+ extracted_tool_calls = jamba_tool_parser.extract_tool_calls(
90
+ model_output, request=None) # type: ignore[arg-type]
91
+ assert not extracted_tool_calls.tools_called
92
+ assert extracted_tool_calls.tool_calls == []
93
+ assert extracted_tool_calls.content == model_output
94
+
95
+
96
+@pytest.mark.parametrize(
97
+ ids=[
98
+ "single_tool",
99
+ "single_tool_with_content",
100
+ "parallel_tools",
101
+ ],
102
+ argnames=["model_output", "expected_tool_calls", "expected_content"],
103
+ argvalues=[
104
+ (
105
+ ''' <tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}\n]</tool_calls>''', # noqa: E501
106
+ [
107
+ ToolCall(function=FunctionCall(name="get_current_weather",
108
+ arguments=json.dumps(
109
+ {
110
+ "city": "Dallas",
111
+ "state": "TX",
112
+ "unit": "fahrenheit"
113
+ })))
114
+ ],
115
+ None),
116
+ (
117
+ ''' Sure! let me call the tool for you.<tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}\n]</tool_calls>''', # noqa: E501
118
+ [
119
+ ToolCall(function=FunctionCall(name="get_current_weather",
120
+ arguments=json.dumps(
121
+ {
122
+ "city": "Dallas",
123
+ "state": "TX",
124
+ "unit": "fahrenheit"
125
+ })))
126
+ ],
127
+ " Sure! let me call the tool for you."),
128
+ (
129
+ ''' <tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}},\n {"name": "get_current_weather", "arguments": {"city": "Orlando", "state": "FL", "unit": "fahrenheit"}}\n]</tool_calls>''', # noqa: E501
130
+ [
131
+ ToolCall(function=FunctionCall(name="get_current_weather",
132
+ arguments=json.dumps(
133
+ {
134
+ "city": "Dallas",
135
+ "state": "TX",
136
+ "unit": "fahrenheit"
137
+ }))),
138
+ ToolCall(function=FunctionCall(name="get_current_weather",
139
+ arguments=json.dumps(
140
+ {
141
+ "city": "Orlando",
142
+ "state": "FL",
143
+ "unit": "fahrenheit"
144
+ })))
145
+ ],
146
+ None)
147
+ ],
148
+)
149
+def test_extract_tool_calls(jamba_tool_parser, model_output,
150
+ expected_tool_calls, expected_content):
151
+ extracted_tool_calls = jamba_tool_parser.extract_tool_calls(
152
+ model_output, request=None) # type: ignore[arg-type]
153
+ assert extracted_tool_calls.tools_called
154
+
155
+ assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls)
156
+
157
+ assert extracted_tool_calls.content == expected_content
158
+
159
+
160
+@pytest.mark.parametrize(
161
+ ids=[
162
+ "no_tools",
163
+ "single_tool",
164
+ "single_tool_with_content",
165
+ "parallel_tools",
166
+ ],
167
+ argnames=["model_output", "expected_tool_calls", "expected_content"],
168
+ argvalues=[
169
+ ('''This is a test''', [], '''This is a test'''),
170
+ (
171
+ ''' <tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}\n]</tool_calls>''', # noqa: E501
172
+ [
173
+ ToolCall(function=FunctionCall(name="get_current_weather",
174
+ arguments=json.dumps(
175
+ {
176
+ "city": "Dallas",
177
+ "state": "TX",
178
+ "unit": "fahrenheit"
179
+ })))
180
+ ],
181
+ " "),
182
+ (
183
+ ''' Sure! let me call the tool for you.<tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}\n]</tool_calls>''', # noqa: E501
184
+ [
185
+ ToolCall(function=FunctionCall(name="get_current_weather",
186
+ arguments=json.dumps(
187
+ {
188
+ "city": "Dallas",
189
+ "state": "TX",
190
+ "unit": "fahrenheit"
191
+ })))
192
+ ],
193
+ " Sure! let me call the tool for you."),
194
+ (
195
+ ''' <tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}},\n {"name": "get_current_weather", "arguments": {"city": "Orlando", "state": "FL", "unit": "fahrenheit"}}\n]</tool_calls>''', # noqa: E501
196
+ [
197
+ ToolCall(function=FunctionCall(name="get_current_weather",
198
+ arguments=json.dumps(
199
+ {
200
+ "city": "Dallas",
201
+ "state": "TX",
202
+ "unit": "fahrenheit"
203
+ }))),
204
+ ToolCall(function=FunctionCall(name="get_current_weather",
205
+ arguments=json.dumps(
206
+ {
207
+ "city": "Orlando",
208
+ "state": "FL",
209
+ "unit": "fahrenheit"
210
+ })))
211
+ ],
212
+ " ")
213
+ ],
214
+)
215
+def test_extract_tool_calls_streaming(jamba_tool_parser, jamba_tokenizer,
216
+ model_output, expected_tool_calls,
217
+ expected_content):
218
+ other_content: str = ''
219
+ function_names: List[str] = []
220
+ function_args_strs: List[str] = []
221
+ tool_call_idx: int = -1
222
+ tool_call_ids: List[Optional[str]] = []
223
+
224
+ for delta_message in stream_delta_message_generator(
225
+ jamba_tool_parser, jamba_tokenizer, model_output):
226
+ # role should never be streamed from tool parser
227
+ assert not delta_message.role
228
+
229
+ if delta_message.content:
230
+ other_content += delta_message.content
231
+
232
+ streamed_tool_calls = delta_message.tool_calls
233
+
234
+ if streamed_tool_calls and len(streamed_tool_calls) > 0:
235
+ # make sure only one diff is present - correct even for parallel
236
+ assert len(streamed_tool_calls) == 1
237
+ tool_call = streamed_tool_calls[0]
238
+
239
+ # if a new tool is being called, set up empty arguments
240
+ if tool_call.index != tool_call_idx:
241
+ tool_call_idx = tool_call.index
242
+ function_args_strs.append("")
243
+ tool_call_ids.append(None)
244
+
245
+ # if a tool call ID is streamed, make sure one hasn't been already
246
+ if tool_call.id and not tool_call_ids[tool_call.index]:
247
+ tool_call_ids[tool_call.index] = tool_call.id
248
+
249
+ # if parts of the function start being streamed
250
+ if tool_call.function:
251
+ # if the function name is defined, set it. it should be streamed
252
+ # IN ENTIRETY, exactly one time.
253
+ if tool_call.function.name:
254
+ assert isinstance(tool_call.function.name, str)
255
+ function_names.append(tool_call.function.name)
256
+
257
+ if tool_call.function.arguments:
258
+ # make sure they're a string and then add them to the list
259
+ assert isinstance(tool_call.function.arguments, str)
260
+
261
+ function_args_strs[
262
+ tool_call.index] += tool_call.function.arguments
263
+
264
+ assert other_content == expected_content
265
+
266
+ actual_tool_calls = [
267
+ ToolCall(id=tool_call_id,
268
+ function=FunctionCall(
269
+ name=function_name,
270
+ arguments=partial_json_parser.ensure_json(
271
+ function_args_str, Allow.OBJ | Allow.STR)))
272
+ for tool_call_id, function_name, function_args_str in zip(
273
+ tool_call_ids, function_names, function_args_strs)
274
+ ]
275
+ assert_tool_calls(actual_tool_calls, expected_tool_calls)
vllm/entrypoints/openai/tool_parsers/__init__.py
+3
-1
@@ -1,10 +1,12 @@
1
from .abstract_tool_parser import ToolParser, ToolParserManager
2
from .hermes_tool_parser import Hermes2ProToolParser
3
from .internlm2_tool_parser import Internlm2ToolParser
4
+from .jamba_tool_parser import JambaToolParser
5
from .llama_tool_parser import Llama3JsonToolParser
6
from .mistral_tool_parser import MistralToolParser
7
8
__all__ = [
9
"ToolParser", "ToolParserManager", "Hermes2ProToolParser",
9
- "MistralToolParser", "Internlm2ToolParser", "Llama3JsonToolParser"
10
+ "MistralToolParser", "Internlm2ToolParser", "Llama3JsonToolParser",
11
+ "JambaToolParser"
12
]
vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+2
-1
@@ -53,7 +53,8 @@ class Hermes2ProToolParser(ToolParser):
53
self.tool_call_start_token_id = self.vocab.get(
54
self.tool_call_start_token)
55
self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
56
- if not self.tool_call_start_token_id or not self.tool_call_end_token_id:
56
+ if (self.tool_call_start_token_id is None
57
+ or self.tool_call_end_token_id is None):
58
raise RuntimeError(
59
"Hermes 2 Pro Tool parser could not locate tool call start/end "
60
"tokens in the tokenizer!")
vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py
new
+300
@@ -0,0 +1,300 @@
1
+import json
2
+import re
3
+from typing import Dict, List, Sequence, Union
4
+
5
+import partial_json_parser
6
+from partial_json_parser.core.options import Allow
7
+
8
+from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
9
+ DeltaFunctionCall, DeltaMessage,
10
+ DeltaToolCall,
11
+ ExtractedToolCallInformation,
12
+ FunctionCall, ToolCall)
13
+from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
14
+from vllm.entrypoints.openai.tool_parsers.utils import (
15
+ extract_intermediate_diff)
16
+from vllm.logger import init_logger
17
+from vllm.transformers_utils.tokenizer import AnyTokenizer
18
+from vllm.transformers_utils.tokenizers import MistralTokenizer
19
+from vllm.utils import random_uuid
20
+
21
+logger = init_logger(__name__)
22
+
23
+
24
+@ToolParserManager.register_module("jamba")
25
+class JambaToolParser(ToolParser):
26
+
27
+ def __init__(self, tokenizer: AnyTokenizer):
28
+ super().__init__(tokenizer)
29
+
30
+ if isinstance(self.model_tokenizer, MistralTokenizer):
31
+ raise ValueError(
32
+ "Detected a MistralTokenizer tokenizer when using a Jamba model"
33
+ )
34
+
35
+ self.current_tool_name_sent: bool = False
36
+ self.prev_tool_call_arr: List[Dict] = []
37
+ self.current_tool_id: int = -1
38
+ self.streamed_args_for_tool: List[str] = [
39
+ ] # map what has been streamed for each tool so far to a list
40
+
41
+ self.tool_calls_start_token: str = "<tool_calls>"
42
+ self.tool_calls_end_token: str = "</tool_calls>"
43
+
44
+ self.tool_calls_regex = re.compile(
45
+ rf"{self.tool_calls_start_token}(.*?){self.tool_calls_end_token}",
46
+ re.DOTALL)
47
+
48
+ if not self.model_tokenizer:
49
+ raise ValueError(
50
+ "The model tokenizer must be passed to the ToolParser "
51
+ "constructor during construction.")
52
+ self.tool_calls_start_token_id = self.vocab.get(
53
+ self.tool_calls_start_token)
54
+ self.tool_calls_end_token_id = self.vocab.get(
55
+ self.tool_calls_end_token)
56
+ if (self.tool_calls_start_token_id is None
57
+ or self.tool_calls_end_token_id is None):
58
+ raise RuntimeError(
59
+ "Jamba Tool parser could not locate tool calls start/end "
60
+ "tokens in the tokenizer!")
61
+
62
+ def adjust_request(
63
+ self, request: ChatCompletionRequest) -> ChatCompletionRequest:
64
+ if request.tools and request.tool_choice != 'none':
65
+ # do not skip special tokens because jamba use the special
66
+ # tokens to indicate the start and end of the tool calls
67
+ # information.
68
+ request.skip_special_tokens = False
69
+ return request
70
+
71
+ def extract_tool_calls(
72
+ self, model_output: str,
73
+ request: ChatCompletionRequest) -> ExtractedToolCallInformation:
74
+
75
+ # sanity check; avoid unnecessary processing
76
+ if self.tool_calls_start_token not in model_output:
77
+ return ExtractedToolCallInformation(tools_called=False,
78
+ tool_calls=[],
79
+ content=model_output)
80
+
81
+ else:
82
+
83
+ try:
84
+ # use a regex to find the tool call between the tags
85
+ function_calls = self.tool_calls_regex.findall(model_output)[0]
86
+
87
+ # load the JSON, and then use it to build the Function and
88
+ # Tool Call
89
+ raw_function_calls = json.loads(function_calls)
90
+ tool_calls = [
91
+ ToolCall(
92
+ type="function",
93
+ function=FunctionCall(
94
+ name=function_call["name"],
95
+ # function call args are JSON but as a string
96
+ arguments=json.dumps(function_call["arguments"])))
97
+ for function_call in raw_function_calls
98
+ ]
99
+
100
+ content = model_output[:model_output.
101
+ find(self.tool_calls_start_token)]
102
+ return ExtractedToolCallInformation(
103
+ tools_called=True,
104
+ tool_calls=tool_calls,
105
+ content=content if
106
+ (len(content) > 0 and content != " ") else None)
107
+
108
+ except Exception:
109
+ logger.exception(
110
+ "Error in extracting tool call from response.")
111
+ return ExtractedToolCallInformation(tools_called=False,
112
+ tool_calls=[],
113
+ content=model_output)
114
+
115
+ def extract_tool_calls_streaming(
116
+ self,
117
+ previous_text: str,
118
+ current_text: str,
119
+ delta_text: str,
120
+ previous_token_ids: Sequence[int],
121
+ current_token_ids: Sequence[int],
122
+ delta_token_ids: Sequence[int],
123
+ request: ChatCompletionRequest,
124
+ ) -> Union[DeltaMessage, None]:
125
+
126
+ # if the tool call token is not in the tokens generated so far, append
127
+ # output to contents since it's not a tool
128
+ if self.tool_calls_start_token not in current_text:
129
+ return DeltaMessage(content=delta_text)
130
+
131
+ # if the tool call token ID IS in the tokens generated so far, that
132
+ # means we're parsing as tool calls now
133
+
134
+ # handle if we detected the start of tool calls token which means
135
+ # the start of tool calling
136
+ if (self.tool_calls_start_token_id in delta_token_ids
137
+ and len(delta_token_ids) == 1):
138
+ # if it's the only token, return None, so we don't send a chat
139
+ # completion and don't send a control token
140
+ return None
141
+
142
+ # bit mask flags for partial JSON parsing. If the name hasn't been
143
+ # sent yet, don't allow sending
144
+ # an incomplete string since OpenAI only ever (as far as I have
145
+ # seen) allows sending the entire tool/ function name at once.
146
+ flags = Allow.ALL if self.current_tool_name_sent \
147
+ else Allow.ALL & ~Allow.STR
148
+ try:
149
+
150
+ # Extract the tool calls between the special tool call tokens
151
+ parsable_arr = current_text.split(
152
+ self.tool_calls_start_token)[-1].split(
153
+ self.tool_calls_end_token)[0]
154
+
155
+ # tool calls are generated in an array, so do partial JSON
156
+ # parsing on the entire array
157
+ try:
158
+ tool_call_arr: List[Dict] = partial_json_parser.loads(
159
+ parsable_arr, flags)
160
+ except partial_json_parser.core.exceptions.MalformedJSON:
161
+ logger.debug('not enough tokens to parse into JSON yet')
162
+ return None
163
+
164
+ # select as the current tool call the one we're on the state at
165
+
166
+ current_tool_call: Dict = tool_call_arr[self.current_tool_id] \
167
+ if len(tool_call_arr) > 0 else {}
168
+
169
+ # case -- if no tokens have been streamed for the tool, e.g.
170
+ # only the array brackets, stream nothing
171
+ if len(tool_call_arr) == 0:
172
+ return None
173
+
174
+ # case: we are starting a new tool in the array
175
+ # -> array has > 0 length AND length has moved past cursor
176
+ elif (len(tool_call_arr) > 0
177
+ and len(tool_call_arr) > self.current_tool_id + 1):
178
+
179
+ # if we're moving on to a new call, first make sure we
180
+ # haven't missed anything in the previous one that was
181
+ # auto-generated due to JSON completions, but wasn't
182
+ # streamed to the client yet.
183
+ if self.current_tool_id >= 0:
184
+ diff: Union[str, None] = current_tool_call.get("arguments")
185
+
186
+ if diff:
187
+ diff = json.dumps(diff).replace(
188
+ self.streamed_args_for_tool[self.current_tool_id],
189
+ "")
190
+ delta = DeltaMessage(tool_calls=[
191
+ DeltaToolCall(index=self.current_tool_id,
192
+ function=DeltaFunctionCall(
193
+ arguments=diff).model_dump(
194
+ exclude_none=True))
195
+ ])
196
+ self.streamed_args_for_tool[
197
+ self.current_tool_id] += diff
198
+ else:
199
+ delta = None
200
+ else:
201
+ delta = None
202
+ # re-set stuff pertaining to progress in the current tool
203
+ self.current_tool_id = len(tool_call_arr) - 1
204
+ self.current_tool_name_sent = False
205
+ self.streamed_args_for_tool.append("")
206
+ logger.debug("starting on new tool %d", self.current_tool_id)
207
+ return delta
208
+
209
+ # case: update an existing tool - this is handled below
210
+
211
+ # if the current tool name hasn't been sent, send if available
212
+ # - otherwise send nothing
213
+ if not self.current_tool_name_sent:
214
+ function_name = current_tool_call.get("name")
215
+ if function_name:
216
+
217
+ delta = DeltaMessage(tool_calls=[
218
+ DeltaToolCall(index=self.current_tool_id,
219
+ type="function",
220
+ id=f"chatcmpl-tool-{random_uuid()}",
221
+ function=DeltaFunctionCall(
222
+ name=function_name).model_dump(
223
+ exclude_none=True))
224
+ ])
225
+ self.current_tool_name_sent = True
226
+ else:
227
+ delta = None
228
+
229
+ # now we know we're on the same tool call and we're streaming
230
+ # arguments
231
+ else:
232
+
233
+ prev_arguments = self.prev_tool_call_arr[
234
+ self.current_tool_id].get("arguments")
235
+ cur_arguments = current_tool_call.get("arguments")
236
+
237
+ new_text = delta_text.replace("\'", "\"")
238
+
239
+ if not cur_arguments and not prev_arguments:
240
+
241
+ delta = None
242
+ elif not cur_arguments and prev_arguments:
243
+ logger.error(
244
+ "INVARIANT - impossible to have arguments reset "
245
+ "mid-arguments")
246
+ delta = None
247
+ elif cur_arguments and not prev_arguments:
248
+ cur_arguments_json = json.dumps(cur_arguments)
249
+ logger.debug("finding %s in %s", new_text,
250
+ cur_arguments_json)
251
+
252
+ arguments_delta = cur_arguments_json[:cur_arguments_json.
253
+ index(new_text) +
254
+ len(new_text)]
255
+ logger.debug("First tokens in arguments received: %s",
256
+ arguments_delta)
257
+ delta = DeltaMessage(tool_calls=[
258
+ DeltaToolCall(index=self.current_tool_id,
259
+ function=DeltaFunctionCall(
260
+ arguments=arguments_delta).
261
+ model_dump(exclude_none=True))
262
+ ])
263
+ self.streamed_args_for_tool[
264
+ self.current_tool_id] += arguments_delta
265
+
266
+ elif cur_arguments and prev_arguments:
267
+ cur_args_json = json.dumps(cur_arguments)
268
+ prev_args_json = json.dumps(prev_arguments)
269
+ logger.debug("Searching for diff between \n%s\n%s",
270
+ cur_args_json, prev_args_json)
271
+
272
+ argument_diff = extract_intermediate_diff(
273
+ cur_args_json, prev_args_json)
274
+ logger.debug("got arguments diff: %s", argument_diff)
275
+ delta = DeltaMessage(tool_calls=[
276
+ DeltaToolCall(index=self.current_tool_id,
277
+ function=DeltaFunctionCall(
278
+ arguments=argument_diff).model_dump(
279
+ exclude_none=True))
280
+ ])
281
+ self.streamed_args_for_tool[
282
+ self.current_tool_id] += argument_diff
283
+ else:
284
+ # try parsing it with regular JSON - if it works we're
285
+ # at the end, and we need to send the difference between
286
+ # tokens streamed so far and the valid JSON
287
+ delta = None
288
+
289
+ # check to see if the name is defined and has been sent. if so,
290
+ # stream the name - otherwise keep waiting
291
+ # finish by setting old and returning None as base case
292
+ self.prev_tool_call_arr = tool_call_arr
293
+ return delta
294
+
295
+ except Exception:
296
+ logger.exception("Error trying to handle streaming tool call.")
297
+ logger.debug(
298
+ "Skipping chunk as a result of tool streaming extraction "
299
+ "error")
300
+ return None
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+1
-1
@@ -63,7 +63,7 @@ class MistralToolParser(ToolParser):
63
self.bot_token = "[TOOL_CALLS]"
64
self.bot_token_id = self.vocab.get(self.bot_token)
65
self.tool_call_regex = re.compile(r"\[{.*?}\]", re.DOTALL)
66
- if not self.bot_token_id:
66
+ if self.bot_token_id is None:
67
raise RuntimeError(
68
"Mistral Tool Parser could not locate the tool call token in "
69
"the tokenizer!")