@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f028dff33d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[BugFix] Fix hermes tool parser output error stream arguments in some cases (#10395) (#10398)

Signed-off-by: xiyuan lee <lixiyuan@haier.com>

COSMOPlat committed Nov 19, 2024 at 21:42 UTC f028dff33d3d0b0dfe71e0e0354b355b8232a4ec
1 file changed +12 -13
vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+12 -13
@@ -12,8 +12,6 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12 FunctionCall, ToolCall)
13 from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
14 ToolParser, ToolParserManager)
15 -from vllm.entrypoints.openai.tool_parsers.utils import (
16 - extract_intermediate_diff)
15 from vllm.logger import init_logger
16 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
17 from vllm.utils import random_uuid
@@ -190,8 +188,11 @@ class Hermes2ProToolParser(ToolParser):
188 diff = self.prev_tool_call_arr[self.current_tool_id].get(
189 "arguments")
190 if diff:
193 - diff = json.dumps(diff).replace(
194 - self.streamed_args_for_tool[self.current_tool_id], "")
191 + diff = diff.encode('utf-8').decode(
192 + 'unicode_escape') if diff is str else diff
193 + diff = json.dumps(
194 + diff, ensure_ascii=False
195 + )[len(self.streamed_args_for_tool[self.current_tool_id]):]
196 logger.debug(
197 "Finishing tool and found diff that had not "
198 "been streamed yet: %s", diff)
@@ -307,22 +308,20 @@ class Hermes2ProToolParser(ToolParser):
308
309 # last case -- we have an update to existing arguments.
310 elif cur_arguments and prev_arguments:
311 + if isinstance(delta_text, str) and len(delta_text.rstrip(
312 + )) >= 1 and delta_text.rstrip()[-1] == '}':
313 + delta_text = delta_text.rstrip()[:-1]
314 +
315 + logger.debug("got diff %s", delta_text)
316
311 - cur_args_json = json.dumps(cur_arguments)
312 - prev_args_json = json.dumps(prev_arguments)
313 - logger.debug("Searching for diff between\n%s", cur_args_json)
314 - logger.debug("and\n%s", prev_args_json)
315 - argument_diff = extract_intermediate_diff(
316 - cur_args_json, prev_args_json)
317 - logger.debug("got argument diff %s", argument_diff)
317 delta = DeltaMessage(tool_calls=[
318 DeltaToolCall(index=self.current_tool_id,
319 function=DeltaFunctionCall(
321 - arguments=argument_diff).model_dump(
320 + arguments=delta_text).model_dump(
321 exclude_none=True))
322 ])
323 self.streamed_args_for_tool[self.current_tool_id] \
325 - += argument_diff
324 + += delta_text
325
326 # handle saving the state for the current tool into
327 # the "prev" list for use in diffing for the next iteration