@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d2b1bf55ec
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Frontend][Feature] Add jamba tool parser (#9154)

tomeras91 committed Oct 18, 2024 at 13:27 UTC d2b1bf55ec0d50f76762b902ca84036ac53e9646
6 files changed +595 -9
docs/source/serving/openai_compatible_server.md
+14 -6
@@ -157,7 +157,7 @@ vLLM will use guided decoding to ensure the response matches the tool parameter
157 To enable this feature, you should set the following flags:
158 * `--enable-auto-tool-choice` -- **mandatory** Auto tool choice. tells vLLM that you want to enable the model to generate its own tool calls when it
159 deems appropriate.
160 -* `--tool-call-parser` -- select the tool parser to use - currently either `hermes` or `mistral` or `llama3_json` or `internlm`. Additional tool parsers
160 +* `--tool-call-parser` -- select the tool parser to use (listed below). Additional tool parsers
161 will continue to be added in the future, and also can register your own tool parsers in the `--tool-parser-plugin`.
162 * `--tool-parser-plugin` -- **optional** tool parser plugin used to register user defined tool parsers into vllm, the registered tool parser name can be specified in `--tool-call-parser`.
163 * `--chat-template` -- **optional** for auto tool choice. the path to the chat template which handles `tool`-role messages and `assistant`-role messages
@@ -168,7 +168,7 @@ from HuggingFace; and you can find an example of this in a `tokenizer_config.jso
168
169 If your favorite tool-calling model is not supported, please feel free to contribute a parser & tool use chat template!
170
171 -#### Hermes Models
171 +#### Hermes Models (`hermes`)
172 All Nous Research Hermes-series models newer than Hermes 2 Pro should be supported.
173 * `NousResearch/Hermes-2-Pro-*`
174 * `NousResearch/Hermes-2-Theta-*`
@@ -180,7 +180,7 @@ step in their creation_.
180
181 Flags: `--tool-call-parser hermes`
182
183 -#### Mistral Models
183 +#### Mistral Models (`mistral`)
184 Supported models:
185 * `mistralai/Mistral-7B-Instruct-v0.3` (confirmed)
186 * Additional mistral function-calling models are compatible as well.
@@ -199,7 +199,7 @@ when tools are provided, that results in much better reliability when working wi
199
200 Recommended flags: `--tool-call-parser mistral --chat-template examples/tool_chat_template_mistral_parallel.jinja`
201
202 -#### Llama Models
202 +#### Llama Models (`llama3_json`)
203 Supported models:
204 * `meta-llama/Meta-Llama-3.1-8B-Instruct`
205 * `meta-llama/Meta-Llama-3.1-70B-Instruct`
@@ -219,16 +219,24 @@ it works better with vLLM.
219
220 Recommended flags: `--tool-call-parser llama3_json --chat-template examples/tool_chat_template_llama3_json.jinja`
221
222 -#### Internlm Models
222 +#### InternLM Models (`internlm`)
223 Supported models:
224 * `internlm/internlm2_5-7b-chat` (confirmed)
225 * Additional internlm2.5 function-calling models are compatible as well
226
227 Known issues:
228 -* Although this implementation also supports Internlm2, the tool call results are not stable when testing with the `internlm/internlm2-chat-7b` model.
228 +* Although this implementation also supports InternLM2, the tool call results are not stable when testing with the `internlm/internlm2-chat-7b` model.
229
230 Recommended flags: `--tool-call-parser internlm --chat-template examples/tool_chat_template_internlm2_tool.jinja`
231
232 +#### Jamba Models (`jamba`)
233 +AI21's Jamba-1.5 models are supported.
234 +* `ai21labs/AI21-Jamba-1.5-Mini`
235 +* `ai21labs/AI21-Jamba-1.5-Large`
236 +
237 +
238 +Flags: `--tool-call-parser jamba`
239 +
240
241 ### How to write a tool parser plugin
242
tests/tool_use/test_jamba_tool_parser.py new
+275
@@ -0,0 +1,275 @@
1 +import json
2 +from typing import Generator, List, Optional
3 +
4 +import partial_json_parser
5 +import pytest
6 +from partial_json_parser.core.options import Allow
7 +
8 +from vllm.entrypoints.openai.protocol import (DeltaMessage, FunctionCall,
9 + ToolCall)
10 +from vllm.entrypoints.openai.tool_parsers import JambaToolParser
11 +from vllm.transformers_utils.detokenizer import detokenize_incrementally
12 +from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
13 +
14 +MODEL = "ai21labs/Jamba-tiny-dev"
15 +
16 +
17 +@pytest.fixture(scope="module")
18 +def jamba_tokenizer():
19 + return get_tokenizer(tokenizer_name=MODEL)
20 +
21 +
22 +@pytest.fixture
23 +def jamba_tool_parser(jamba_tokenizer):
24 + return JambaToolParser(jamba_tokenizer)
25 +
26 +
27 +def assert_tool_calls(actual_tool_calls: List[ToolCall],
28 + expected_tool_calls: List[ToolCall]):
29 + assert len(actual_tool_calls) == len(expected_tool_calls)
30 +
31 + for actual_tool_call, expected_tool_call in zip(actual_tool_calls,
32 + expected_tool_calls):
33 + assert isinstance(actual_tool_call.id, str)
34 + assert len(actual_tool_call.id) > 16
35 +
36 + assert actual_tool_call.type == "function"
37 + assert actual_tool_call.function == expected_tool_call.function
38 +
39 +
40 +def stream_delta_message_generator(
41 + jamba_tool_parser: JambaToolParser, jamba_tokenizer: AnyTokenizer,
42 + model_output: str) -> Generator[DeltaMessage, None, None]:
43 + all_token_ids = jamba_tokenizer.encode(model_output,
44 + add_special_tokens=False)
45 +
46 + previous_text = ""
47 + previous_tokens = None
48 + prefix_offset = 0
49 + read_offset = 0
50 + for i, delta_token in enumerate(all_token_ids):
51 + delta_token_ids = [delta_token]
52 + previous_token_ids = all_token_ids[:i]
53 + current_token_ids = all_token_ids[:i + 1]
54 +
55 + (new_tokens, delta_text, new_prefix_offset,
56 + new_read_offset) = detokenize_incrementally(
57 + tokenizer=jamba_tokenizer,
58 + all_input_ids=current_token_ids,
59 + prev_tokens=previous_tokens,
60 + prefix_offset=prefix_offset,
61 + read_offset=read_offset,
62 + skip_special_tokens=False,
63 + spaces_between_special_tokens=True,
64 + )
65 +
66 + current_text = previous_text + delta_text
67 +
68 + delta_message = jamba_tool_parser.extract_tool_calls_streaming(
69 + previous_text,
70 + current_text,
71 + delta_text,
72 + previous_token_ids,
73 + current_token_ids,
74 + delta_token_ids,
75 + request=None, # type: ignore[arg-type]
76 + )
77 + if delta_message:
78 + yield delta_message
79 +
80 + previous_text = current_text
81 + previous_tokens = previous_tokens + new_tokens if previous_tokens\
82 + else new_tokens
83 + prefix_offset = new_prefix_offset
84 + read_offset = new_read_offset
85 +
86 +
87 +def test_extract_tool_calls_no_tools(jamba_tool_parser):
88 + model_output = "This is a test"
89 + extracted_tool_calls = jamba_tool_parser.extract_tool_calls(
90 + model_output, request=None) # type: ignore[arg-type]
91 + assert not extracted_tool_calls.tools_called
92 + assert extracted_tool_calls.tool_calls == []
93 + assert extracted_tool_calls.content == model_output
94 +
95 +
96 +@pytest.mark.parametrize(
97 + ids=[
98 + "single_tool",
99 + "single_tool_with_content",
100 + "parallel_tools",
101 + ],
102 + argnames=["model_output", "expected_tool_calls", "expected_content"],
103 + argvalues=[
104 + (
105 + ''' <tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}\n]</tool_calls>''', # noqa: E501
106 + [
107 + ToolCall(function=FunctionCall(name="get_current_weather",
108 + arguments=json.dumps(
109 + {
110 + "city": "Dallas",
111 + "state": "TX",
112 + "unit": "fahrenheit"
113 + })))
114 + ],
115 + None),
116 + (
117 + ''' Sure! let me call the tool for you.<tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}\n]</tool_calls>''', # noqa: E501
118 + [
119 + ToolCall(function=FunctionCall(name="get_current_weather",
120 + arguments=json.dumps(
121 + {
122 + "city": "Dallas",
123 + "state": "TX",
124 + "unit": "fahrenheit"
125 + })))
126 + ],
127 + " Sure! let me call the tool for you."),
128 + (
129 + ''' <tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}},\n {"name": "get_current_weather", "arguments": {"city": "Orlando", "state": "FL", "unit": "fahrenheit"}}\n]</tool_calls>''', # noqa: E501
130 + [
131 + ToolCall(function=FunctionCall(name="get_current_weather",
132 + arguments=json.dumps(
133 + {
134 + "city": "Dallas",
135 + "state": "TX",
136 + "unit": "fahrenheit"
137 + }))),
138 + ToolCall(function=FunctionCall(name="get_current_weather",
139 + arguments=json.dumps(
140 + {
141 + "city": "Orlando",
142 + "state": "FL",
143 + "unit": "fahrenheit"
144 + })))
145 + ],
146 + None)
147 + ],
148 +)
149 +def test_extract_tool_calls(jamba_tool_parser, model_output,
150 + expected_tool_calls, expected_content):
151 + extracted_tool_calls = jamba_tool_parser.extract_tool_calls(
152 + model_output, request=None) # type: ignore[arg-type]
153 + assert extracted_tool_calls.tools_called
154 +
155 + assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls)
156 +
157 + assert extracted_tool_calls.content == expected_content
158 +
159 +
160 +@pytest.mark.parametrize(
161 + ids=[
162 + "no_tools",
163 + "single_tool",
164 + "single_tool_with_content",
165 + "parallel_tools",
166 + ],
167 + argnames=["model_output", "expected_tool_calls", "expected_content"],
168 + argvalues=[
169 + ('''This is a test''', [], '''This is a test'''),
170 + (
171 + ''' <tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}\n]</tool_calls>''', # noqa: E501
172 + [
173 + ToolCall(function=FunctionCall(name="get_current_weather",
174 + arguments=json.dumps(
175 + {
176 + "city": "Dallas",
177 + "state": "TX",
178 + "unit": "fahrenheit"
179 + })))
180 + ],
181 + " "),
182 + (
183 + ''' Sure! let me call the tool for you.<tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}\n]</tool_calls>''', # noqa: E501
184 + [
185 + ToolCall(function=FunctionCall(name="get_current_weather",
186 + arguments=json.dumps(
187 + {
188 + "city": "Dallas",
189 + "state": "TX",
190 + "unit": "fahrenheit"
191 + })))
192 + ],
193 + " Sure! let me call the tool for you."),
194 + (
195 + ''' <tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}},\n {"name": "get_current_weather", "arguments": {"city": "Orlando", "state": "FL", "unit": "fahrenheit"}}\n]</tool_calls>''', # noqa: E501
196 + [
197 + ToolCall(function=FunctionCall(name="get_current_weather",
198 + arguments=json.dumps(
199 + {
200 + "city": "Dallas",
201 + "state": "TX",
202 + "unit": "fahrenheit"
203 + }))),
204 + ToolCall(function=FunctionCall(name="get_current_weather",
205 + arguments=json.dumps(
206 + {
207 + "city": "Orlando",
208 + "state": "FL",
209 + "unit": "fahrenheit"
210 + })))
211 + ],
212 + " ")
213 + ],
214 +)
215 +def test_extract_tool_calls_streaming(jamba_tool_parser, jamba_tokenizer,
216 + model_output, expected_tool_calls,
217 + expected_content):
218 + other_content: str = ''
219 + function_names: List[str] = []
220 + function_args_strs: List[str] = []
221 + tool_call_idx: int = -1
222 + tool_call_ids: List[Optional[str]] = []
223 +
224 + for delta_message in stream_delta_message_generator(
225 + jamba_tool_parser, jamba_tokenizer, model_output):
226 + # role should never be streamed from tool parser
227 + assert not delta_message.role
228 +
229 + if delta_message.content:
230 + other_content += delta_message.content
231 +
232 + streamed_tool_calls = delta_message.tool_calls
233 +
234 + if streamed_tool_calls and len(streamed_tool_calls) > 0:
235 + # make sure only one diff is present - correct even for parallel
236 + assert len(streamed_tool_calls) == 1
237 + tool_call = streamed_tool_calls[0]
238 +
239 + # if a new tool is being called, set up empty arguments
240 + if tool_call.index != tool_call_idx:
241 + tool_call_idx = tool_call.index
242 + function_args_strs.append("")
243 + tool_call_ids.append(None)
244 +
245 + # if a tool call ID is streamed, make sure one hasn't been already
246 + if tool_call.id and not tool_call_ids[tool_call.index]:
247 + tool_call_ids[tool_call.index] = tool_call.id
248 +
249 + # if parts of the function start being streamed
250 + if tool_call.function:
251 + # if the function name is defined, set it. it should be streamed
252 + # IN ENTIRETY, exactly one time.
253 + if tool_call.function.name:
254 + assert isinstance(tool_call.function.name, str)
255 + function_names.append(tool_call.function.name)
256 +
257 + if tool_call.function.arguments:
258 + # make sure they're a string and then add them to the list
259 + assert isinstance(tool_call.function.arguments, str)
260 +
261 + function_args_strs[
262 + tool_call.index] += tool_call.function.arguments
263 +
264 + assert other_content == expected_content
265 +
266 + actual_tool_calls = [
267 + ToolCall(id=tool_call_id,
268 + function=FunctionCall(
269 + name=function_name,
270 + arguments=partial_json_parser.ensure_json(
271 + function_args_str, Allow.OBJ | Allow.STR)))
272 + for tool_call_id, function_name, function_args_str in zip(
273 + tool_call_ids, function_names, function_args_strs)
274 + ]
275 + assert_tool_calls(actual_tool_calls, expected_tool_calls)
vllm/entrypoints/openai/tool_parsers/__init__.py
+3 -1
@@ -1,10 +1,12 @@
1 from .abstract_tool_parser import ToolParser, ToolParserManager
2 from .hermes_tool_parser import Hermes2ProToolParser
3 from .internlm2_tool_parser import Internlm2ToolParser
4 +from .jamba_tool_parser import JambaToolParser
5 from .llama_tool_parser import Llama3JsonToolParser
6 from .mistral_tool_parser import MistralToolParser
7
8 __all__ = [
9 "ToolParser", "ToolParserManager", "Hermes2ProToolParser",
9 - "MistralToolParser", "Internlm2ToolParser", "Llama3JsonToolParser"
10 + "MistralToolParser", "Internlm2ToolParser", "Llama3JsonToolParser",
11 + "JambaToolParser"
12 ]
vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+2 -1
@@ -53,7 +53,8 @@ class Hermes2ProToolParser(ToolParser):
53 self.tool_call_start_token_id = self.vocab.get(
54 self.tool_call_start_token)
55 self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
56 - if not self.tool_call_start_token_id or not self.tool_call_end_token_id:
56 + if (self.tool_call_start_token_id is None
57 + or self.tool_call_end_token_id is None):
58 raise RuntimeError(
59 "Hermes 2 Pro Tool parser could not locate tool call start/end "
60 "tokens in the tokenizer!")
vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py new
+300
@@ -0,0 +1,300 @@
1 +import json
2 +import re
3 +from typing import Dict, List, Sequence, Union
4 +
5 +import partial_json_parser
6 +from partial_json_parser.core.options import Allow
7 +
8 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
9 + DeltaFunctionCall, DeltaMessage,
10 + DeltaToolCall,
11 + ExtractedToolCallInformation,
12 + FunctionCall, ToolCall)
13 +from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
14 +from vllm.entrypoints.openai.tool_parsers.utils import (
15 + extract_intermediate_diff)
16 +from vllm.logger import init_logger
17 +from vllm.transformers_utils.tokenizer import AnyTokenizer
18 +from vllm.transformers_utils.tokenizers import MistralTokenizer
19 +from vllm.utils import random_uuid
20 +
21 +logger = init_logger(__name__)
22 +
23 +
24 +@ToolParserManager.register_module("jamba")
25 +class JambaToolParser(ToolParser):
26 +
27 + def __init__(self, tokenizer: AnyTokenizer):
28 + super().__init__(tokenizer)
29 +
30 + if isinstance(self.model_tokenizer, MistralTokenizer):
31 + raise ValueError(
32 + "Detected a MistralTokenizer tokenizer when using a Jamba model"
33 + )
34 +
35 + self.current_tool_name_sent: bool = False
36 + self.prev_tool_call_arr: List[Dict] = []
37 + self.current_tool_id: int = -1
38 + self.streamed_args_for_tool: List[str] = [
39 + ] # map what has been streamed for each tool so far to a list
40 +
41 + self.tool_calls_start_token: str = "<tool_calls>"
42 + self.tool_calls_end_token: str = "</tool_calls>"
43 +
44 + self.tool_calls_regex = re.compile(
45 + rf"{self.tool_calls_start_token}(.*?){self.tool_calls_end_token}",
46 + re.DOTALL)
47 +
48 + if not self.model_tokenizer:
49 + raise ValueError(
50 + "The model tokenizer must be passed to the ToolParser "
51 + "constructor during construction.")
52 + self.tool_calls_start_token_id = self.vocab.get(
53 + self.tool_calls_start_token)
54 + self.tool_calls_end_token_id = self.vocab.get(
55 + self.tool_calls_end_token)
56 + if (self.tool_calls_start_token_id is None
57 + or self.tool_calls_end_token_id is None):
58 + raise RuntimeError(
59 + "Jamba Tool parser could not locate tool calls start/end "
60 + "tokens in the tokenizer!")
61 +
62 + def adjust_request(
63 + self, request: ChatCompletionRequest) -> ChatCompletionRequest:
64 + if request.tools and request.tool_choice != 'none':
65 + # do not skip special tokens because jamba use the special
66 + # tokens to indicate the start and end of the tool calls
67 + # information.
68 + request.skip_special_tokens = False
69 + return request
70 +
71 + def extract_tool_calls(
72 + self, model_output: str,
73 + request: ChatCompletionRequest) -> ExtractedToolCallInformation:
74 +
75 + # sanity check; avoid unnecessary processing
76 + if self.tool_calls_start_token not in model_output:
77 + return ExtractedToolCallInformation(tools_called=False,
78 + tool_calls=[],
79 + content=model_output)
80 +
81 + else:
82 +
83 + try:
84 + # use a regex to find the tool call between the tags
85 + function_calls = self.tool_calls_regex.findall(model_output)[0]
86 +
87 + # load the JSON, and then use it to build the Function and
88 + # Tool Call
89 + raw_function_calls = json.loads(function_calls)
90 + tool_calls = [
91 + ToolCall(
92 + type="function",
93 + function=FunctionCall(
94 + name=function_call["name"],
95 + # function call args are JSON but as a string
96 + arguments=json.dumps(function_call["arguments"])))
97 + for function_call in raw_function_calls
98 + ]
99 +
100 + content = model_output[:model_output.
101 + find(self.tool_calls_start_token)]
102 + return ExtractedToolCallInformation(
103 + tools_called=True,
104 + tool_calls=tool_calls,
105 + content=content if
106 + (len(content) > 0 and content != " ") else None)
107 +
108 + except Exception:
109 + logger.exception(
110 + "Error in extracting tool call from response.")
111 + return ExtractedToolCallInformation(tools_called=False,
112 + tool_calls=[],
113 + content=model_output)
114 +
115 + def extract_tool_calls_streaming(
116 + self,
117 + previous_text: str,
118 + current_text: str,
119 + delta_text: str,
120 + previous_token_ids: Sequence[int],
121 + current_token_ids: Sequence[int],
122 + delta_token_ids: Sequence[int],
123 + request: ChatCompletionRequest,
124 + ) -> Union[DeltaMessage, None]:
125 +
126 + # if the tool call token is not in the tokens generated so far, append
127 + # output to contents since it's not a tool
128 + if self.tool_calls_start_token not in current_text:
129 + return DeltaMessage(content=delta_text)
130 +
131 + # if the tool call token ID IS in the tokens generated so far, that
132 + # means we're parsing as tool calls now
133 +
134 + # handle if we detected the start of tool calls token which means
135 + # the start of tool calling
136 + if (self.tool_calls_start_token_id in delta_token_ids
137 + and len(delta_token_ids) == 1):
138 + # if it's the only token, return None, so we don't send a chat
139 + # completion and don't send a control token
140 + return None
141 +
142 + # bit mask flags for partial JSON parsing. If the name hasn't been
143 + # sent yet, don't allow sending
144 + # an incomplete string since OpenAI only ever (as far as I have
145 + # seen) allows sending the entire tool/ function name at once.
146 + flags = Allow.ALL if self.current_tool_name_sent \
147 + else Allow.ALL & ~Allow.STR
148 + try:
149 +
150 + # Extract the tool calls between the special tool call tokens
151 + parsable_arr = current_text.split(
152 + self.tool_calls_start_token)[-1].split(
153 + self.tool_calls_end_token)[0]
154 +
155 + # tool calls are generated in an array, so do partial JSON
156 + # parsing on the entire array
157 + try:
158 + tool_call_arr: List[Dict] = partial_json_parser.loads(
159 + parsable_arr, flags)
160 + except partial_json_parser.core.exceptions.MalformedJSON:
161 + logger.debug('not enough tokens to parse into JSON yet')
162 + return None
163 +
164 + # select as the current tool call the one we're on the state at
165 +
166 + current_tool_call: Dict = tool_call_arr[self.current_tool_id] \
167 + if len(tool_call_arr) > 0 else {}
168 +
169 + # case -- if no tokens have been streamed for the tool, e.g.
170 + # only the array brackets, stream nothing
171 + if len(tool_call_arr) == 0:
172 + return None
173 +
174 + # case: we are starting a new tool in the array
175 + # -> array has > 0 length AND length has moved past cursor
176 + elif (len(tool_call_arr) > 0
177 + and len(tool_call_arr) > self.current_tool_id + 1):
178 +
179 + # if we're moving on to a new call, first make sure we
180 + # haven't missed anything in the previous one that was
181 + # auto-generated due to JSON completions, but wasn't
182 + # streamed to the client yet.
183 + if self.current_tool_id >= 0:
184 + diff: Union[str, None] = current_tool_call.get("arguments")
185 +
186 + if diff:
187 + diff = json.dumps(diff).replace(
188 + self.streamed_args_for_tool[self.current_tool_id],
189 + "")
190 + delta = DeltaMessage(tool_calls=[
191 + DeltaToolCall(index=self.current_tool_id,
192 + function=DeltaFunctionCall(
193 + arguments=diff).model_dump(
194 + exclude_none=True))
195 + ])
196 + self.streamed_args_for_tool[
197 + self.current_tool_id] += diff
198 + else:
199 + delta = None
200 + else:
201 + delta = None
202 + # re-set stuff pertaining to progress in the current tool
203 + self.current_tool_id = len(tool_call_arr) - 1
204 + self.current_tool_name_sent = False
205 + self.streamed_args_for_tool.append("")
206 + logger.debug("starting on new tool %d", self.current_tool_id)
207 + return delta
208 +
209 + # case: update an existing tool - this is handled below
210 +
211 + # if the current tool name hasn't been sent, send if available
212 + # - otherwise send nothing
213 + if not self.current_tool_name_sent:
214 + function_name = current_tool_call.get("name")
215 + if function_name:
216 +
217 + delta = DeltaMessage(tool_calls=[
218 + DeltaToolCall(index=self.current_tool_id,
219 + type="function",
220 + id=f"chatcmpl-tool-{random_uuid()}",
221 + function=DeltaFunctionCall(
222 + name=function_name).model_dump(
223 + exclude_none=True))
224 + ])
225 + self.current_tool_name_sent = True
226 + else:
227 + delta = None
228 +
229 + # now we know we're on the same tool call and we're streaming
230 + # arguments
231 + else:
232 +
233 + prev_arguments = self.prev_tool_call_arr[
234 + self.current_tool_id].get("arguments")
235 + cur_arguments = current_tool_call.get("arguments")
236 +
237 + new_text = delta_text.replace("\'", "\"")
238 +
239 + if not cur_arguments and not prev_arguments:
240 +
241 + delta = None
242 + elif not cur_arguments and prev_arguments:
243 + logger.error(
244 + "INVARIANT - impossible to have arguments reset "
245 + "mid-arguments")
246 + delta = None
247 + elif cur_arguments and not prev_arguments:
248 + cur_arguments_json = json.dumps(cur_arguments)
249 + logger.debug("finding %s in %s", new_text,
250 + cur_arguments_json)
251 +
252 + arguments_delta = cur_arguments_json[:cur_arguments_json.
253 + index(new_text) +
254 + len(new_text)]
255 + logger.debug("First tokens in arguments received: %s",
256 + arguments_delta)
257 + delta = DeltaMessage(tool_calls=[
258 + DeltaToolCall(index=self.current_tool_id,
259 + function=DeltaFunctionCall(
260 + arguments=arguments_delta).
261 + model_dump(exclude_none=True))
262 + ])
263 + self.streamed_args_for_tool[
264 + self.current_tool_id] += arguments_delta
265 +
266 + elif cur_arguments and prev_arguments:
267 + cur_args_json = json.dumps(cur_arguments)
268 + prev_args_json = json.dumps(prev_arguments)
269 + logger.debug("Searching for diff between \n%s\n%s",
270 + cur_args_json, prev_args_json)
271 +
272 + argument_diff = extract_intermediate_diff(
273 + cur_args_json, prev_args_json)
274 + logger.debug("got arguments diff: %s", argument_diff)
275 + delta = DeltaMessage(tool_calls=[
276 + DeltaToolCall(index=self.current_tool_id,
277 + function=DeltaFunctionCall(
278 + arguments=argument_diff).model_dump(
279 + exclude_none=True))
280 + ])
281 + self.streamed_args_for_tool[
282 + self.current_tool_id] += argument_diff
283 + else:
284 + # try parsing it with regular JSON - if it works we're
285 + # at the end, and we need to send the difference between
286 + # tokens streamed so far and the valid JSON
287 + delta = None
288 +
289 + # check to see if the name is defined and has been sent. if so,
290 + # stream the name - otherwise keep waiting
291 + # finish by setting old and returning None as base case
292 + self.prev_tool_call_arr = tool_call_arr
293 + return delta
294 +
295 + except Exception:
296 + logger.exception("Error trying to handle streaming tool call.")
297 + logger.debug(
298 + "Skipping chunk as a result of tool streaming extraction "
299 + "error")
300 + return None
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+1 -1
@@ -63,7 +63,7 @@ class MistralToolParser(ToolParser):
63 self.bot_token = "[TOOL_CALLS]"
64 self.bot_token_id = self.vocab.get(self.bot_token)
65 self.tool_call_regex = re.compile(r"\[{.*?}\]", re.DOTALL)
66 - if not self.bot_token_id:
66 + if self.bot_token_id is None:
67 raise RuntimeError(
68 "Mistral Tool Parser could not locate the tool call token in "
69 "the tokenizer!")