A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Frontend] Added support for HermesToolParser for models without special tokens (#16890)
Signed-off-by: minpeter <kali2005611@gmail.com>
Woonggi Min committed
Aug 17, 2025 at 02:38 UTC
68373d3126b4d2c49a9983fe0696bbd48fc8aad7
2 files changed
+191
-17
tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py
new
+127
@@ -0,0 +1,127 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import json
5
+
6
+import pytest
7
+
8
+from ....utils import RemoteOpenAIServer
9
+
10
+MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
11
+LORA_MODEL = "minpeter/LoRA-Llama-3.2-1B-tool-vllm-ci"
12
+
13
+SERVER_ARGS = [
14
+ "--enforce-eager",
15
+ "--enable-auto-tool-choice",
16
+ "--tool-call-parser",
17
+ "hermes",
18
+ "--enable-lora",
19
+ "--lora-modules",
20
+ f"{LORA_MODEL}={LORA_MODEL}",
21
+]
22
+
23
+TOOLS = [{
24
+ "type": "function",
25
+ "function": {
26
+ "name": "get_current_weather",
27
+ "description": "Get the current weather in a given location",
28
+ "parameters": {
29
+ "type": "object",
30
+ "properties": {
31
+ "location": {
32
+ "type": "string",
33
+ "description":
34
+ "The city and state, e.g. San Francisco, CA",
35
+ },
36
+ "unit": {
37
+ "type": "string",
38
+ "enum": ["celsius", "fahrenheit"]
39
+ },
40
+ },
41
+ "required": ["location"],
42
+ },
43
+ },
44
+}]
45
+
46
+MESSAGES = [{"role": "user", "content": "What's the weather like in Boston?"}]
47
+
48
+
49
+@pytest.mark.asyncio
50
+async def test_non_streaming_tool_call():
51
+ """Test tool call in non-streaming mode."""
52
+ with RemoteOpenAIServer(MODEL_NAME, SERVER_ARGS) as server:
53
+ client = server.get_async_client()
54
+
55
+ response = await client.chat.completions.create(
56
+ model=LORA_MODEL,
57
+ messages=MESSAGES,
58
+ tools=TOOLS,
59
+ tool_choice="auto",
60
+ temperature=0.0,
61
+ )
62
+
63
+ assert response.choices
64
+ choice = response.choices[0]
65
+ message = choice.message
66
+
67
+ assert choice.finish_reason == "tool_calls"
68
+ assert message.tool_calls is not None
69
+
70
+ tool_call = message.tool_calls[0]
71
+ assert tool_call.type == "function"
72
+ assert tool_call.function.name == "get_current_weather"
73
+
74
+ arguments = json.loads(tool_call.function.arguments)
75
+ assert "location" in arguments
76
+ assert "Boston" in arguments["location"]
77
+ print("\n[Non-Streaming Test Passed]")
78
+ print(f"Tool Call: {tool_call.function.name}")
79
+ print(f"Arguments: {arguments}")
80
+
81
+
82
+@pytest.mark.asyncio
83
+async def test_streaming_tool_call():
84
+ """Test tool call in streaming mode."""
85
+ with RemoteOpenAIServer(MODEL_NAME, SERVER_ARGS) as server:
86
+ client = server.get_async_client()
87
+
88
+ stream = await client.chat.completions.create(
89
+ model=LORA_MODEL,
90
+ messages=MESSAGES,
91
+ tools=TOOLS,
92
+ tool_choice="auto",
93
+ temperature=0.0,
94
+ stream=True,
95
+ )
96
+
97
+ tool_call_chunks = {}
98
+ async for chunk in stream:
99
+ if not chunk.choices:
100
+ continue
101
+
102
+ delta = chunk.choices[0].delta
103
+ if not delta or not delta.tool_calls:
104
+ continue
105
+
106
+ for tool_chunk in delta.tool_calls:
107
+ index = tool_chunk.index
108
+ if index not in tool_call_chunks:
109
+ tool_call_chunks[index] = {"name": "", "arguments": ""}
110
+
111
+ if tool_chunk.function.name:
112
+ tool_call_chunks[index]["name"] += tool_chunk.function.name
113
+ if tool_chunk.function.arguments:
114
+ tool_call_chunks[index][
115
+ "arguments"] += tool_chunk.function.arguments
116
+
117
+ assert len(tool_call_chunks) == 1
118
+ reconstructed_tool_call = tool_call_chunks[0]
119
+
120
+ assert reconstructed_tool_call["name"] == "get_current_weather"
121
+
122
+ arguments = json.loads(reconstructed_tool_call["arguments"])
123
+ assert "location" in arguments
124
+ assert "Boston" in arguments["location"]
125
+ print("\n[Streaming Test Passed]")
126
+ print(f"Reconstructed Tool Call: {reconstructed_tool_call['name']}")
127
+ print(f"Reconstructed Arguments: {arguments}")
vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+64
-17
@@ -52,14 +52,51 @@ class Hermes2ProToolParser(ToolParser):
52
raise ValueError(
53
"The model tokenizer must be passed to the ToolParser "
54
"constructor during construction.")
55
- self.tool_call_start_token_id = self.vocab.get(
56
- self.tool_call_start_token)
57
- self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
58
- if (self.tool_call_start_token_id is None
59
- or self.tool_call_end_token_id is None):
60
- raise RuntimeError(
61
- "Hermes 2 Pro Tool parser could not locate tool call start/end "
62
- "tokens in the tokenizer!")
55
+ self.tool_call_start_token_ids = self.model_tokenizer.encode(
56
+ self.tool_call_start_token, add_special_tokens=False)
57
+ self.tool_call_end_token_ids = self.model_tokenizer.encode(
58
+ self.tool_call_end_token, add_special_tokens=False)
59
+
60
+ self.tool_call_start_token_array = [
61
+ self.model_tokenizer.decode([token_id])
62
+ for token_id in self.tool_call_start_token_ids
63
+ ]
64
+
65
+ self.tool_call_end_token_array = [
66
+ self.model_tokenizer.decode([token_id])
67
+ for token_id in self.tool_call_end_token_ids
68
+ ]
69
+
70
+ self.buffered_delta_text = ""
71
+
72
+ # Very simple idea: when encountering tokens like <, tool, _call, >,
73
+ # <, /, tool, _call, >, store them in a buffer.
74
+ # When the last token is encountered, empty the buffer and return it.
75
+ # If a token appears in an incorrect sequence while storing in the buffer,
76
+ # return the preceding buffer along with the token.
77
+ def tool_call_delta_buffer(self, delta_text: str):
78
+ # If the sequence of tool_call_start or tool_call_end tokens is not yet
79
+ # complete, fill the buffer with the token and return "".
80
+ if (delta_text in self.tool_call_start_token_array
81
+ or delta_text in self.tool_call_end_token_array):
82
+ # If delta_text is the last token of tool_call_start_token or
83
+ # tool_call_end_token, empty the buffer and return
84
+ # the buffered text + delta_text.
85
+ if (delta_text == self.tool_call_start_token_array[-1]
86
+ or delta_text == self.tool_call_end_token_array[-1]):
87
+ buffered_text = self.buffered_delta_text
88
+ self.buffered_delta_text = ""
89
+ return buffered_text + delta_text
90
+ else:
91
+ self.buffered_delta_text = self.buffered_delta_text + delta_text
92
+ return ""
93
+ else:
94
+ if self.buffered_delta_text:
95
+ buffered_text = self.buffered_delta_text
96
+ self.buffered_delta_text = ""
97
+ return buffered_text + delta_text
98
+ else:
99
+ return delta_text
100
101
def extract_tool_calls(
102
self,
@@ -124,11 +161,23 @@ class Hermes2ProToolParser(ToolParser):
161
delta_token_ids: Sequence[int],
162
request: ChatCompletionRequest,
163
) -> Union[DeltaMessage, None]:
164
+ # 1. All tokens are parsed based on _text, not token_ids.
165
+ # 2. All incoming text data is processed by the tool_call_delta_buffer
166
+ # function for buffering before being used for parsing.
167
+
168
+ delta_text = self.tool_call_delta_buffer(delta_text)
169
+ # If the last characters of previous_text
170
+ # match self.buffered_delta_text, remove only the matching part.
171
+ if (len(previous_text) >= len(self.buffered_delta_text)
172
+ and previous_text[-len(self.buffered_delta_text):]
173
+ == self.buffered_delta_text):
174
+ previous_text = previous_text[:-len(self.buffered_delta_text)]
175
+ current_text = previous_text + delta_text
176
177
logger.debug("delta_text: %s", delta_text)
178
logger.debug("delta_token_ids: %s", delta_token_ids)
179
# check to see if we should be streaming a tool call - is there a
131
- if self.tool_call_start_token_id not in current_token_ids:
180
+ if self.tool_call_start_token not in current_text:
181
logger.debug("No tool call tokens found!")
182
return DeltaMessage(content=delta_text)
183
@@ -136,14 +185,12 @@ class Hermes2ProToolParser(ToolParser):
185
186
# figure out where we are in the parsing by counting tool call
187
# start & end tags
139
- prev_tool_start_count = previous_token_ids.count(
140
- self.tool_call_start_token_id)
141
- prev_tool_end_count = previous_token_ids.count(
142
- self.tool_call_end_token_id)
143
- cur_tool_start_count = current_token_ids.count(
144
- self.tool_call_start_token_id)
145
- cur_tool_end_count = current_token_ids.count(
146
- self.tool_call_end_token_id)
188
+ prev_tool_start_count = previous_text.count(
189
+ self.tool_call_start_token)
190
+ prev_tool_end_count = previous_text.count(self.tool_call_end_token)
191
+ cur_tool_start_count = current_text.count(
192
+ self.tool_call_start_token)
193
+ cur_tool_end_count = current_text.count(self.tool_call_end_token)
194
tool_call_portion = None
195
text_portion = None
196