A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Access `get_vocab` instead of `vocab` in tool parsers (#9188)
Cyrus Leung committed
Oct 9, 2024 at 22:59 UTC
cfaa6008e666d4e9bb5131ece68f8609b6f94ee4
3 files changed
+11
-6
vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py
+7
@@ -1,6 +1,7 @@
1
import importlib
2
import importlib.util
3
import os
4
+from functools import cached_property
5
from typing import Callable, Dict, List, Optional, Sequence, Type, Union
6
7
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
@@ -29,6 +30,12 @@ class ToolParser:
30
31
self.model_tokenizer = tokenizer
32
33
+ @cached_property
34
+ def vocab(self) -> Dict[str, int]:
35
+ # NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab
36
+ # whereas all tokenizers have .get_vocab()
37
+ return self.model_tokenizer.get_vocab()
38
+
39
def adjust_request(
40
self, request: ChatCompletionRequest) -> ChatCompletionRequest:
41
"""
vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+3
-4
@@ -50,10 +50,9 @@ class Hermes2ProToolParser(ToolParser):
50
raise ValueError(
51
"The model tokenizer must be passed to the ToolParser "
52
"constructor during construction.")
53
- self.tool_call_start_token_id: int = self.model_tokenizer.vocab.get(
54
- self.tool_call_start_token, None)
55
- self.tool_call_end_token_id: int = self.model_tokenizer.vocab.get(
56
- self.tool_call_end_token, None)
53
+ self.tool_call_start_token_id = self.vocab.get(
54
+ self.tool_call_start_token)
55
+ self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
56
if not self.tool_call_start_token_id or not self.tool_call_end_token_id:
57
raise RuntimeError(
58
"Hermes 2 Pro Tool parser could not locate tool call start/end "
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+1
-2
@@ -61,8 +61,7 @@ class MistralToolParser(ToolParser):
61
self.streamed_args_for_tool: List[str] = [
62
] # map what has been streamed for each tool so far to a list
63
self.bot_token = "[TOOL_CALLS]"
64
- self.bot_token_id = self.model_tokenizer.get_vocab().get(
65
- self.bot_token, None)
64
+ self.bot_token_id = self.vocab.get(self.bot_token)
65
self.tool_call_regex = re.compile(r"\[{.*?}\]", re.DOTALL)
66
if not self.bot_token_id:
67
raise RuntimeError(