@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cfaa6008e6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] Access `get_vocab` instead of `vocab` in tool parsers (#9188)

Cyrus Leung committed Oct 9, 2024 at 22:59 UTC cfaa6008e666d4e9bb5131ece68f8609b6f94ee4
3 files changed +11 -6
vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py
+7
@@ -1,6 +1,7 @@
1 import importlib
2 import importlib.util
3 import os
4 +from functools import cached_property
5 from typing import Callable, Dict, List, Optional, Sequence, Type, Union
6
7 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
@@ -29,6 +30,12 @@ class ToolParser:
30
31 self.model_tokenizer = tokenizer
32
33 + @cached_property
34 + def vocab(self) -> Dict[str, int]:
35 + # NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab
36 + # whereas all tokenizers have .get_vocab()
37 + return self.model_tokenizer.get_vocab()
38 +
39 def adjust_request(
40 self, request: ChatCompletionRequest) -> ChatCompletionRequest:
41 """
vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+3 -4
@@ -50,10 +50,9 @@ class Hermes2ProToolParser(ToolParser):
50 raise ValueError(
51 "The model tokenizer must be passed to the ToolParser "
52 "constructor during construction.")
53 - self.tool_call_start_token_id: int = self.model_tokenizer.vocab.get(
54 - self.tool_call_start_token, None)
55 - self.tool_call_end_token_id: int = self.model_tokenizer.vocab.get(
56 - self.tool_call_end_token, None)
53 + self.tool_call_start_token_id = self.vocab.get(
54 + self.tool_call_start_token)
55 + self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
56 if not self.tool_call_start_token_id or not self.tool_call_end_token_id:
57 raise RuntimeError(
58 "Hermes 2 Pro Tool parser could not locate tool call start/end "
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+1 -2
@@ -61,8 +61,7 @@ class MistralToolParser(ToolParser):
61 self.streamed_args_for_tool: List[str] = [
62 ] # map what has been streamed for each tool so far to a list
63 self.bot_token = "[TOOL_CALLS]"
64 - self.bot_token_id = self.model_tokenizer.get_vocab().get(
65 - self.bot_token, None)
64 + self.bot_token_id = self.vocab.get(self.bot_token)
65 self.tool_call_regex = re.compile(r"\[{.*?}\]", re.DOTALL)
66 if not self.bot_token_id:
67 raise RuntimeError(