A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[BugFix] Fix hermes tool parser output error stream arguments in some cases (#10395) (#10398)
Signed-off-by: xiyuan lee <lixiyuan@haier.com>
COSMOPlat committed
Nov 19, 2024 at 21:42 UTC
f028dff33d3d0b0dfe71e0e0354b355b8232a4ec
1 file changed
+12
-13
vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+12
-13
@@ -12,8 +12,6 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12
FunctionCall, ToolCall)
13
from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
14
ToolParser, ToolParserManager)
15
-from vllm.entrypoints.openai.tool_parsers.utils import (
16
- extract_intermediate_diff)
15
from vllm.logger import init_logger
16
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
17
from vllm.utils import random_uuid
@@ -190,8 +188,11 @@ class Hermes2ProToolParser(ToolParser):
188
diff = self.prev_tool_call_arr[self.current_tool_id].get(
189
"arguments")
190
if diff:
193
- diff = json.dumps(diff).replace(
194
- self.streamed_args_for_tool[self.current_tool_id], "")
191
+ diff = diff.encode('utf-8').decode(
192
+ 'unicode_escape') if diff is str else diff
193
+ diff = json.dumps(
194
+ diff, ensure_ascii=False
195
+ )[len(self.streamed_args_for_tool[self.current_tool_id]):]
196
logger.debug(
197
"Finishing tool and found diff that had not "
198
"been streamed yet: %s", diff)
@@ -307,22 +308,20 @@ class Hermes2ProToolParser(ToolParser):
308
309
# last case -- we have an update to existing arguments.
310
elif cur_arguments and prev_arguments:
311
+ if isinstance(delta_text, str) and len(delta_text.rstrip(
312
+ )) >= 1 and delta_text.rstrip()[-1] == '}':
313
+ delta_text = delta_text.rstrip()[:-1]
314
+
315
+ logger.debug("got diff %s", delta_text)
316
311
- cur_args_json = json.dumps(cur_arguments)
312
- prev_args_json = json.dumps(prev_arguments)
313
- logger.debug("Searching for diff between\n%s", cur_args_json)
314
- logger.debug("and\n%s", prev_args_json)
315
- argument_diff = extract_intermediate_diff(
316
- cur_args_json, prev_args_json)
317
- logger.debug("got argument diff %s", argument_diff)
317
delta = DeltaMessage(tool_calls=[
318
DeltaToolCall(index=self.current_tool_id,
319
function=DeltaFunctionCall(
321
- arguments=argument_diff).model_dump(
320
+ arguments=delta_text).model_dump(
321
exclude_none=True))
322
])
323
self.streamed_args_for_tool[self.current_tool_id] \
325
- += argument_diff
324
+ += delta_text
325
326
# handle saving the state for the current tool into
327
# the "prev" list for use in diffing for the next iteration