@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e39400a4b6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

Fix streaming for granite tool call when <|tool_call|> is present (#11069)

Signed-off-by: Max de Bayser <mbayser@br.ibm.com>

Maximilien de Bayser committed Dec 11, 2024 at 01:51 UTC e39400a4b60d28ff5c0a1a5194068c928adcaf98
1 file changed +5 -1
vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py
+5 -1
@@ -88,7 +88,11 @@ class GraniteToolParser(ToolParser):
88 ) -> Union[DeltaMessage, None]:
89
90 start_idx = consume_space(0, current_text)
91 - if not current_text or current_text[start_idx] != '[':
91 + if current_text[start_idx:].startswith(self.bot_token):
92 + start_idx = consume_space(start_idx + len(self.bot_token),
93 + current_text)
94 + if not current_text or start_idx >= len(current_text)\
95 + or current_text[start_idx] != '[':
96 return DeltaMessage(content=delta_text)
97
98 # bit mask flags for partial JSON parsing. If the name hasn't been