A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 17:30 UTC
[BUG]: fix HF tokenizer concurrent borrow in tool parsers (#40059)
Signed-off-by: Yifan <yzong@redhat.com> Co-authored-by: timon0305 <timon0305@outlook.com> Co-authored-by: sfeng33 <4florafeng@gmail.com>
yzong-rh committed
Apr 23, 2026 at 21:20 UTC
c9bf77df92a3526575756f576127268474068e96
3 files changed
+37
-38
tests/tool_parsers/test_llama3_json_tool_parser.py
+10
-3
@@ -4,15 +4,22 @@
4
from unittest.mock import MagicMock, patch
5
6
import pytest
7
+from transformers import AutoTokenizer
8
9
from vllm.entrypoints.openai.engine.protocol import ExtractedToolCallInformation
9
-from vllm.tokenizers import TokenizerLike
10
from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
11
12
+LLAMA_MODEL = "meta-llama/Llama-3.2-1B-Instruct"
13
+
14
+
15
+@pytest.fixture(scope="module")
16
+def llama_tokenizer():
17
+ return AutoTokenizer.from_pretrained(LLAMA_MODEL)
18
+
19
20
@pytest.fixture
14
-def parser(default_tokenizer: TokenizerLike):
15
- return Llama3JsonToolParser(default_tokenizer)
21
+def parser(llama_tokenizer):
22
+ return Llama3JsonToolParser(llama_tokenizer)
23
24
25
def test_extract_tool_calls_simple(parser):
vllm/tool_parsers/functiongemma_tool_parser.py
+15
-27
@@ -34,6 +34,21 @@ class FunctionGemmaToolParser(ToolParser):
34
<start_function_call>call:func_name{param:<escape>value<escape>}<end_function_call>
35
"""
36
37
+ # FunctionGemma tokens
38
+ tool_call_start_token: str = "<start_function_call>"
39
+ tool_call_end_token: str = "<end_function_call>"
40
+
41
+ # Regex patterns
42
+ tool_call_regex: re.Pattern = re.compile(
43
+ r"<start_function_call>call:(\w+)\{(.*?)\}<end_function_call>"
44
+ r"|<start_function_call>call:(\w+)\{(.*)",
45
+ re.DOTALL,
46
+ )
47
+ arg_regex: re.Pattern = re.compile(
48
+ r"(\w+):<escape>(.*?)<escape>",
49
+ re.DOTALL,
50
+ )
51
+
52
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
53
super().__init__(tokenizer, tools)
54
@@ -42,33 +57,6 @@ class FunctionGemmaToolParser(ToolParser):
57
self.prev_tool_call_arr: list[dict] = []
58
self.current_tool_id: int = -1
59
self.streamed_args_for_tool: list[str] = []
45
-
46
- # FunctionGemma tokens
47
- self.tool_call_start_token: str = "<start_function_call>"
48
- self.tool_call_end_token: str = "<end_function_call>"
49
-
50
- # Regex patterns
51
- self.tool_call_regex = re.compile(
52
- r"<start_function_call>call:(\w+)\{(.*?)\}<end_function_call>"
53
- r"|<start_function_call>call:(\w+)\{(.*)",
54
- re.DOTALL,
55
- )
56
- self.arg_regex = re.compile(
57
- r"(\w+):<escape>(.*?)<escape>",
58
- re.DOTALL,
59
- )
60
-
61
- if self.model_tokenizer:
62
- self.tool_call_start_token_ids = self.model_tokenizer.encode(
63
- self.tool_call_start_token, add_special_tokens=False
64
- )
65
- self.tool_call_end_token_ids = self.model_tokenizer.encode(
66
- self.tool_call_end_token, add_special_tokens=False
67
- )
68
- else:
69
- self.tool_call_start_token_ids = []
70
- self.tool_call_end_token_ids = []
71
-
60
self.buffered_delta_text = ""
61
62
def _parse_arguments(self, args_str: str) -> dict:
vllm/tool_parsers/llama_tool_parser.py
+12
-8
@@ -45,6 +45,12 @@ class Llama3JsonToolParser(ToolParser):
45
llama4_json are set.
46
"""
47
48
+ bot_token: str = "<|python_tag|>"
49
+ # Simple regex to find opening braces - we'll use JSON decoder for parsing
50
+ # This handles arbitrary nesting depth correctly
51
+ tool_call_start_regex: re.Pattern = re.compile(r"\{")
52
+ json_decoder: json.JSONDecoder = json.JSONDecoder()
53
+
54
def __init__(
55
self,
56
tokenizer: PreTrainedTokenizerBase,
@@ -60,14 +66,12 @@ class Llama3JsonToolParser(ToolParser):
66
self.streamed_args_for_tool: list[
67
str
68
] = [] # map what has been streamed for each tool so far to a list
63
- self.bot_token = "<|python_tag|>"
64
- self.bot_token_id = tokenizer.encode(self.bot_token, add_special_tokens=False)[
65
- 0
66
- ]
67
- # Simple regex to find opening braces - we'll use JSON decoder for parsing
68
- # This handles arbitrary nesting depth correctly
69
- self.tool_call_start_regex = re.compile(r"\{")
70
- self.json_decoder = json.JSONDecoder()
69
+ self.bot_token_id = self.vocab.get(self.bot_token)
70
+ if self.bot_token_id is None:
71
+ raise RuntimeError(
72
+ "Llama3JsonToolParser could not locate the bot token "
73
+ f"'{self.bot_token}' in the tokenizer."
74
+ )
75
76
def extract_tool_calls(
77
self, model_output: str, request: ChatCompletionRequest