@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / beb16b2c81
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Handle <|tool_call|> token in granite tool parser (#11039)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>

Travis Johnson committed Dec 10, 2024 at 03:27 UTC beb16b2c810a87b28e7b8a7aa29d26f842f654b9
1 file changed +3 -1
vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py
+3 -1
@@ -35,11 +35,13 @@ class GraniteToolParser(ToolParser):
35
36 def __init__(self, tokenizer: AnyTokenizer):
37 super().__init__(tokenizer)
38 + self.bot_token = "<|tool_call|>"
39
40 def extract_tool_calls(
41 self, model_output: str,
42 request: ChatCompletionRequest) -> ExtractedToolCallInformation:
42 - stripped = model_output.strip()
43 + # remove whitespace and the BOT token if it exists
44 + stripped = model_output.strip().removeprefix(self.bot_token).lstrip()
45 if not stripped or stripped[0] != '[':
46 return ExtractedToolCallInformation(tools_called=False,
47 tool_calls=[],