@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7439a8b5fc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Multiple fixes to tool streaming with hermes and mistral (#10979)

Signed-off-by: cedonley <clayton@donley.io>

Clayton committed Dec 11, 2024 at 17:10 UTC 7439a8b5fcbc4d77bd73496f27d4048c5b43cb22
3 files changed +69 -21
vllm/entrypoints/openai/serving_chat.py
+14 -2
@@ -496,21 +496,33 @@ class OpenAIServingChat(OpenAIServing):
496
497 if self._should_check_for_unstreamed_tool_arg_tokens(
498 delta_message, output) and tool_parser:
499 + latest_delta_len = 0
500 + if ((isinstance(
501 + delta_message.tool_calls[0].function,
502 + DeltaFunctionCall)) and isinstance(
503 + delta_message.tool_calls[0].function.
504 + arguments, str)):
505 + latest_delta_len = len(
506 + delta_message.tool_calls[0].function.
507 + arguments)
508 +
509 # get the expected call based on partial JSON
510 # parsing which "autocompletes" the JSON
511 expected_call = json.dumps(
512 tool_parser.prev_tool_call_arr[index].get(
503 - "arguments", {}))
513 + "arguments", {}),
514 + ensure_ascii=False)
515
516 # get what we've streamed so far for arguments
517 # for the current tool
518 actual_call = tool_parser.streamed_args_for_tool[
519 index]
520 + if (latest_delta_len > 0):
521 + actual_call = actual_call[:-latest_delta_len]
522
523 # check to see if there's anything left to stream
524 remaining_call = expected_call.replace(
525 actual_call, "", 1)
513 -
526 # set that as a delta message
527 delta_message = DeltaMessage(tool_calls=[
528 DeltaToolCall(index=index,
vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+40 -11
@@ -91,7 +91,8 @@ class Hermes2ProToolParser(ToolParser):
91 function=FunctionCall(
92 name=function_call["name"],
93 # function call args are JSON but as a string
94 - arguments=json.dumps(function_call["arguments"])))
94 + arguments=json.dumps(function_call["arguments"],
95 + ensure_ascii=False)))
96 for function_call in raw_function_calls
97 ]
98
@@ -139,13 +140,26 @@ class Hermes2ProToolParser(ToolParser):
140 self.tool_call_start_token_id)
141 cur_tool_end_count = current_token_ids.count(
142 self.tool_call_end_token_id)
143 + tool_call_portion = None
144 + text_portion = None
145
146 # case: if we're generating text, OR rounding out a tool call
147 if (cur_tool_start_count == cur_tool_end_count
145 - and prev_tool_end_count == cur_tool_end_count):
148 + and prev_tool_end_count == cur_tool_end_count
149 + and self.tool_call_end_token not in delta_text):
150 logger.debug("Generating text content! skipping tool parsing.")
147 - if delta_text != self.tool_call_end_token:
148 - return DeltaMessage(content=delta_text)
151 + return DeltaMessage(content=delta_text)
152 +
153 + if self.tool_call_end_token in delta_text:
154 + logger.debug("tool_call_end_token in delta_text")
155 + full_text = current_text + delta_text
156 + tool_call_portion = full_text.split(
157 + self.tool_call_start_token)[-1].split(
158 + self.tool_call_end_token)[0].rstrip()
159 + delta_text = delta_text.split(
160 + self.tool_call_end_token)[0].rstrip()
161 + text_portion = delta_text.split(
162 + self.tool_call_end_token)[-1].lstrip()
163
164 # case: if tool open & close tag counts don't match, we're doing
165 # imaginary "else" block here
@@ -184,15 +198,21 @@ class Hermes2ProToolParser(ToolParser):
198
199 # case -- the current tool call is being closed.
200 elif (cur_tool_start_count == cur_tool_end_count
187 - and cur_tool_end_count > prev_tool_end_count):
201 + and cur_tool_end_count >= prev_tool_end_count):
202 + if (self.prev_tool_call_arr is None
203 + or len(self.prev_tool_call_arr) == 0):
204 + logger.debug(
205 + "attempting to close tool call, but no tool call")
206 + return None
207 diff = self.prev_tool_call_arr[self.current_tool_id].get(
208 "arguments")
209 if diff:
210 diff = diff.encode('utf-8').decode(
211 'unicode_escape') if diff is str else diff
193 - diff = json.dumps(
194 - diff, ensure_ascii=False
195 - )[len(self.streamed_args_for_tool[self.current_tool_id]):]
212 + if ('"}' not in delta_text):
213 + return None
214 + end_loc = delta_text.rindex('"}')
215 + diff = delta_text[:end_loc] + '"}'
216 logger.debug(
217 "Finishing tool and found diff that had not "
218 "been streamed yet: %s", diff)
@@ -221,10 +241,15 @@ class Hermes2ProToolParser(ToolParser):
241 except partial_json_parser.core.exceptions.MalformedJSON:
242 logger.debug('not enough tokens to parse into JSON yet')
243 return None
244 + except json.decoder.JSONDecodeError:
245 + logger.debug("unable to parse JSON")
246 + return None
247
248 # case - we haven't sent the tool name yet. If it's available, send
249 # it. otherwise, wait until it's available.
250 if not self.current_tool_name_sent:
251 + if (current_tool_call is None):
252 + return None
253 function_name: Union[str, None] = current_tool_call.get("name")
254 if function_name:
255 self.current_tool_name_sent = True
@@ -284,13 +309,17 @@ class Hermes2ProToolParser(ToolParser):
309 # autocompleting the JSON
310 elif cur_arguments and not prev_arguments:
311
287 - cur_arguments_json = json.dumps(cur_arguments)
312 + cur_arguments_json = json.dumps(cur_arguments,
313 + ensure_ascii=False)
314 logger.debug("finding %s in %s", delta_text,
315 cur_arguments_json)
316
317 # get the location where previous args differ from current
292 - args_delta_start_loc = cur_arguments_json.index(delta_text) \
293 - + len(delta_text)
318 + if (delta_text not in cur_arguments_json[:-2]):
319 + return None
320 + args_delta_start_loc = cur_arguments_json[:-2]. \
321 + rindex(delta_text) + \
322 + len(delta_text)
323
324 # use that to find the actual delta
325 arguments_delta = cur_arguments_json[:args_delta_start_loc]
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+15 -8
@@ -19,7 +19,6 @@ from vllm.entrypoints.openai.tool_parsers.utils import (
19 extract_intermediate_diff)
20 from vllm.logger import init_logger
21 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
22 -from vllm.utils import random_uuid
22
23 logger = init_logger(__name__)
24
@@ -109,7 +108,8 @@ class MistralToolParser(ToolParser):
108 function=FunctionCall(
109 name=raw_function_call["name"],
110 # function call args are JSON but as a string
112 - arguments=json.dumps(raw_function_call["arguments"])))
111 + arguments=json.dumps(raw_function_call["arguments"],
112 + ensure_ascii=False)))
113 for raw_function_call in function_call_arr
114 ]
115
@@ -199,7 +199,7 @@ class MistralToolParser(ToolParser):
199 diff: Union[str, None] = current_tool_call.get("arguments")
200
201 if diff:
202 - diff = json.dumps(diff).replace(
202 + diff = json.dumps(diff, ensure_ascii=False).replace(
203 self.streamed_args_for_tool[self.current_tool_id],
204 "")
205 delta = DeltaMessage(tool_calls=[
@@ -232,7 +232,7 @@ class MistralToolParser(ToolParser):
232 delta = DeltaMessage(tool_calls=[
233 DeltaToolCall(index=self.current_tool_id,
234 type="function",
235 - id=f"chatcmpl-tool-{random_uuid()}",
235 + id=MistralToolCall.generate_random_id(),
236 function=DeltaFunctionCall(
237 name=function_name).model_dump(
238 exclude_none=True))
@@ -250,6 +250,8 @@ class MistralToolParser(ToolParser):
250 cur_arguments = current_tool_call.get("arguments")
251
252 new_text = delta_text.replace("\'", "\"")
253 + if ('"}' in new_text):
254 + new_text = new_text[:new_text.rindex('"}')]
255
256 if not cur_arguments and not prev_arguments:
257
@@ -260,12 +262,15 @@ class MistralToolParser(ToolParser):
262 "mid-arguments")
263 delta = None
264 elif cur_arguments and not prev_arguments:
263 - cur_arguments_json = json.dumps(cur_arguments)
265 + cur_arguments_json = json.dumps(cur_arguments,
266 + ensure_ascii=False)[:-2]
267 logger.debug("finding %s in %s", new_text,
268 cur_arguments_json)
269
270 + if (new_text not in cur_arguments_json):
271 + return None
272 arguments_delta = cur_arguments_json[:cur_arguments_json.
268 - index(new_text) +
273 + rindex(new_text) +
274 len(new_text)]
275 logger.debug("First tokens in arguments received: %s",
276 arguments_delta)
@@ -279,8 +284,10 @@ class MistralToolParser(ToolParser):
284 self.current_tool_id] += arguments_delta
285
286 elif cur_arguments and prev_arguments:
282 - cur_args_json = json.dumps(cur_arguments)
283 - prev_args_json = json.dumps(prev_arguments)
287 + cur_args_json = json.dumps(cur_arguments,
288 + ensure_ascii=False)
289 + prev_args_json = json.dumps(prev_arguments,
290 + ensure_ascii=False)
291 logger.debug("Searching for diff between \n%s\n%s",
292 cur_args_json, prev_args_json)
293