A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Chore]: Stream tokens vs characters in tool call parser tests (#26513)
Signed-off-by: Ben Browning <bbrownin@redhat.com>
Ben Browning committed
Oct 27, 2025 at 11:06 UTC
3b96f85c36991d9384e8ea5f3d103ef96d932205
6 files changed
+80
-41
tests/entrypoints/openai/tool_parsers/conftest.py
new
+12
@@ -0,0 +1,12 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import pytest
5
+from transformers import AutoTokenizer
6
+
7
+from vllm.transformers_utils.tokenizer import AnyTokenizer
8
+
9
+
10
+@pytest.fixture(scope="function")
11
+def default_tokenizer() -> AnyTokenizer:
12
+ return AutoTokenizer.from_pretrained("gpt2")
tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py
+3
-5
@@ -2,17 +2,15 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import pytest
5
-from transformers import AutoTokenizer
5
6
from vllm.entrypoints.openai.protocol import ExtractedToolCallInformation
7
from vllm.entrypoints.openai.tool_parsers.llama_tool_parser import Llama3JsonToolParser
8
+from vllm.transformers_utils.tokenizer import AnyTokenizer
9
10
11
@pytest.fixture
12
-def parser():
13
- # Use a small tokenizer for testing
14
- tokenizer = AutoTokenizer.from_pretrained("gpt2")
15
- return Llama3JsonToolParser(tokenizer)
12
+def parser(default_tokenizer: AnyTokenizer):
13
+ return Llama3JsonToolParser(default_tokenizer)
14
15
16
def test_extract_tool_calls_simple(parser):
tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py
+12
-12
@@ -11,6 +11,7 @@ from tests.entrypoints.openai.tool_parsers.utils import (
11
)
12
from vllm.entrypoints.openai.protocol import FunctionCall
13
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
14
+from vllm.transformers_utils.tokenizer import AnyTokenizer
15
16
# Test cases similar to pythonic parser but with Llama4 specific format
17
SIMPLE_FUNCTION_OUTPUT = "[get_weather(city='LA', metric='C')]"
@@ -63,10 +64,9 @@ PYTHON_TAG_FUNCTION_OUTPUT = (
64
65
66
@pytest.mark.parametrize("streaming", [True, False])
66
-def test_no_tool_call(streaming: bool):
67
- mock_tokenizer = MagicMock()
67
+def test_no_tool_call(streaming: bool, default_tokenizer: AnyTokenizer):
68
tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(
69
- mock_tokenizer
69
+ default_tokenizer
70
)
71
model_output = "How can I help you today?"
72
@@ -205,11 +205,13 @@ TEST_CASES = [
205
206
@pytest.mark.parametrize("streaming, model_output, expected_tool_calls", TEST_CASES)
207
def test_tool_call(
208
- streaming: bool, model_output: str, expected_tool_calls: list[FunctionCall]
208
+ streaming: bool,
209
+ model_output: str,
210
+ expected_tool_calls: list[FunctionCall],
211
+ default_tokenizer: AnyTokenizer,
212
):
210
- mock_tokenizer = MagicMock()
213
tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(
212
- mock_tokenizer
214
+ default_tokenizer
215
)
216
217
content, tool_calls = run_tool_extraction(
@@ -222,10 +224,9 @@ def test_tool_call(
224
assert actual.function == expected
225
226
225
-def test_streaming_tool_call_with_large_steps():
226
- mock_tokenizer = MagicMock()
227
+def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer):
228
tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(
228
- mock_tokenizer
229
+ default_tokenizer
230
)
231
model_output_deltas = [
232
"<|python_start|>[get_weather(city='LA', metric='C'), "
@@ -245,11 +246,10 @@ def test_streaming_tool_call_with_large_steps():
246
247
248
@pytest.mark.parametrize("streaming", [False])
248
-def test_regex_timeout_handling(streaming: bool):
249
+def test_regex_timeout_handling(streaming: bool, default_tokenizer: AnyTokenizer):
250
"""test regex timeout is handled gracefully"""
250
- mock_tokenizer = MagicMock()
251
tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(
252
- mock_tokenizer
252
+ default_tokenizer
253
)
254
255
fake_problematic_input = "hello world[A(A=" + "\t)A(A=,\t" * 2
tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py
+20
-12
@@ -11,6 +11,7 @@ from tests.entrypoints.openai.tool_parsers.utils import (
11
)
12
from vllm.entrypoints.openai.protocol import FunctionCall
13
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
14
+from vllm.transformers_utils.tokenizer import AnyTokenizer
15
16
# https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#model-response-format-1
17
SIMPLE_FUNCTION_OUTPUT = "get_weather(city='San Francisco', metric='celsius')"
@@ -68,9 +69,10 @@ ESCAPED_STRING_FUNCTION_CALL = FunctionCall(
69
70
71
@pytest.mark.parametrize("streaming", [True, False])
71
-def test_no_tool_call(streaming: bool):
72
- mock_tokenizer = MagicMock()
73
- tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(mock_tokenizer)
72
+def test_no_tool_call(streaming: bool, default_tokenizer: AnyTokenizer):
73
+ tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(
74
+ default_tokenizer
75
+ )
76
model_output = "How can I help you today?"
77
78
content, tool_calls = run_tool_extraction(
@@ -183,10 +185,14 @@ TEST_CASES = [
185
186
@pytest.mark.parametrize("streaming, model_output, expected_tool_calls", TEST_CASES)
187
def test_tool_call(
186
- streaming: bool, model_output: str, expected_tool_calls: list[FunctionCall]
188
+ streaming: bool,
189
+ model_output: str,
190
+ expected_tool_calls: list[FunctionCall],
191
+ default_tokenizer: AnyTokenizer,
192
):
188
- mock_tokenizer = MagicMock()
189
- tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(mock_tokenizer)
193
+ tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(
194
+ default_tokenizer
195
+ )
196
197
content, tool_calls = run_tool_extraction(
198
tool_parser, model_output, streaming=streaming
@@ -199,9 +205,10 @@ def test_tool_call(
205
assert actual.function == expected
206
207
202
-def test_streaming_tool_call_with_large_steps():
203
- mock_tokenizer = MagicMock()
204
- tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(mock_tokenizer)
208
+def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer):
209
+ tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(
210
+ default_tokenizer
211
+ )
212
model_output_deltas = [
213
"<function_calls>get_weather(city='San",
214
" Francisco', metric='celsius')\n"
@@ -221,10 +228,11 @@ def test_streaming_tool_call_with_large_steps():
228
229
230
@pytest.mark.parametrize("streaming", [False])
224
-def test_regex_timeout_handling(streaming: bool):
231
+def test_regex_timeout_handling(streaming: bool, default_tokenizer: AnyTokenizer):
232
"""test regex timeout is handled gracefully"""
226
- mock_tokenizer = MagicMock()
227
- tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(mock_tokenizer)
233
+ tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(
234
+ default_tokenizer
235
+ )
236
237
fake_problematic_input = "hello world[A(A=" + "\t)A(A=,\t" * 2
238
tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py
+12
-12
@@ -11,6 +11,7 @@ from tests.entrypoints.openai.tool_parsers.utils import (
11
)
12
from vllm.entrypoints.openai.protocol import FunctionCall
13
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
14
+from vllm.transformers_utils.tokenizer import AnyTokenizer
15
16
# https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#model-response-format-1
17
SIMPLE_FUNCTION_OUTPUT = "get_weather(city='San Francisco', metric='celsius')"
@@ -60,10 +61,9 @@ ESCAPED_STRING_FUNCTION_CALL = FunctionCall(
61
62
63
@pytest.mark.parametrize("streaming", [True, False])
63
-def test_no_tool_call(streaming: bool):
64
- mock_tokenizer = MagicMock()
64
+def test_no_tool_call(streaming: bool, default_tokenizer: AnyTokenizer):
65
tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(
66
- mock_tokenizer
66
+ default_tokenizer
67
)
68
model_output = "How can I help you today?"
69
@@ -165,11 +165,13 @@ TEST_CASES = [
165
166
@pytest.mark.parametrize("streaming, model_output, expected_tool_calls", TEST_CASES)
167
def test_tool_call(
168
- streaming: bool, model_output: str, expected_tool_calls: list[FunctionCall]
168
+ streaming: bool,
169
+ model_output: str,
170
+ expected_tool_calls: list[FunctionCall],
171
+ default_tokenizer: AnyTokenizer,
172
):
170
- mock_tokenizer = MagicMock()
173
tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(
172
- mock_tokenizer
174
+ default_tokenizer
175
)
176
177
content, tool_calls = run_tool_extraction(
@@ -183,10 +185,9 @@ def test_tool_call(
185
assert actual.function == expected
186
187
186
-def test_streaming_tool_call_with_large_steps():
187
- mock_tokenizer = MagicMock()
188
+def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer):
189
tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(
189
- mock_tokenizer
190
+ default_tokenizer
191
)
192
model_output_deltas = [
193
"[get_weather(city='San",
@@ -207,11 +208,10 @@ def test_streaming_tool_call_with_large_steps():
208
209
210
@pytest.mark.parametrize("streaming", [False])
210
-def test_regex_timeout_handling(streaming: bool):
211
+def test_regex_timeout_handling(streaming: bool, default_tokenizer: AnyTokenizer):
212
"""test regex timeout is handled gracefully"""
212
- mock_tokenizer = MagicMock()
213
tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(
214
- mock_tokenizer
214
+ default_tokenizer
215
)
216
217
fake_problematic_input = "hello world[A(A=" + "\t)A(A=,\t" * 2
tests/entrypoints/openai/tool_parsers/utils.py
+21
@@ -11,6 +11,7 @@ from vllm.entrypoints.openai.protocol import (
11
ToolCall,
12
)
13
from vllm.entrypoints.openai.tool_parsers import ToolParser
14
+from vllm.transformers_utils.tokenizer import AnyTokenizer
15
16
17
class StreamingToolReconstructor:
@@ -110,12 +111,32 @@ def run_tool_extraction_nonstreaming(
111
return tool_parser.extract_tool_calls(model_output, request)
112
113
114
+def split_string_into_token_deltas(tokenizer: AnyTokenizer, text: str) -> list[str]:
115
+ # Split a string into a series of deltas using the provided tokenizer. Each
116
+ # delta will be the string equivalent of a single token.
117
+ token_ids = tokenizer.encode(text, add_special_tokens=False)
118
+ previously_decoded_text = ""
119
+ deltas = []
120
+ for i in range(1, len(token_ids) + 1):
121
+ current_tokens = token_ids[:i]
122
+ current_text = tokenizer.decode(current_tokens)
123
+ new_text = current_text[len(previously_decoded_text) :]
124
+ previously_decoded_text = current_text
125
+ deltas.append(new_text)
126
+ return deltas
127
+
128
+
129
def run_tool_extraction_streaming(
130
tool_parser: ToolParser,
131
model_deltas: Iterable[str],
132
request: ChatCompletionRequest | None = None,
133
assert_one_tool_per_delta: bool = True,
134
) -> StreamingToolReconstructor:
135
+ if isinstance(model_deltas, str):
136
+ model_deltas = split_string_into_token_deltas(
137
+ tool_parser.model_tokenizer, model_deltas
138
+ )
139
+
140
request = request or ChatCompletionRequest(messages=[], model="test-model")
141
reconstructor = StreamingToolReconstructor(
142
assert_one_tool_per_delta=assert_one_tool_per_delta