A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
Fix streaming for granite tool call when <|tool_call|> is present (#11069)
Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
Maximilien de Bayser committed
Dec 11, 2024 at 01:51 UTC
e39400a4b60d28ff5c0a1a5194068c928adcaf98
1 file changed
+5
-1
vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py
+5
-1
@@ -88,7 +88,11 @@ class GraniteToolParser(ToolParser):
88
) -> Union[DeltaMessage, None]:
89
90
start_idx = consume_space(0, current_text)
91
- if not current_text or current_text[start_idx] != '[':
91
+ if current_text[start_idx:].startswith(self.bot_token):
92
+ start_idx = consume_space(start_idx + len(self.bot_token),
93
+ current_text)
94
+ if not current_text or start_idx >= len(current_text)\
95
+ or current_text[start_idx] != '[':
96
return DeltaMessage(content=delta_text)
97
98
# bit mask flags for partial JSON parsing. If the name hasn't been