A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] Handle <|tool_call|> token in granite tool parser (#11039)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
Travis Johnson committed
Dec 10, 2024 at 03:27 UTC
beb16b2c810a87b28e7b8a7aa29d26f842f654b9
1 file changed
+3
-1
vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py
+3
-1
@@ -35,11 +35,13 @@ class GraniteToolParser(ToolParser):
35
36
def __init__(self, tokenizer: AnyTokenizer):
37
super().__init__(tokenizer)
38
+ self.bot_token = "<|tool_call|>"
39
40
def extract_tool_calls(
41
self, model_output: str,
42
request: ChatCompletionRequest) -> ExtractedToolCallInformation:
42
- stripped = model_output.strip()
43
+ # remove whitespace and the BOT token if it exists
44
+ stripped = model_output.strip().removeprefix(self.bot_token).lstrip()
45
if not stripped or stripped[0] != '[':
46
return ExtractedToolCallInformation(tools_called=False,
47
tool_calls=[],