| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import json |
| 5 | |
| 6 | import pytest |
| 7 | |
| 8 | from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest |
| 9 | from vllm.tokenizers import TokenizerLike |
| 10 | from vllm.tool_parsers.abstract_tool_parser import ToolParser |
| 11 | from vllm.tool_parsers.granite4_tool_parser import Granite4ToolParser |
| 12 | from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser |
| 13 | |
| 14 | CONFIGS = { |
| 15 | "llama": { |
| 16 | "tool_parser": Hermes2ProToolParser, |
| 17 | }, |
| 18 | "granite4": { |
| 19 | "tool_parser": Granite4ToolParser, |
| 20 | }, |
| 21 | } |
| 22 | |
| 23 | |
| 24 | @pytest.fixture |
| 25 | def qwen_tokenizer() -> TokenizerLike: |
| 26 | from vllm.tokenizers import get_tokenizer |
| 27 | |
| 28 | return get_tokenizer("Qwen/Qwen3-32B") |
| 29 | |
| 30 | |
| 31 | @pytest.fixture(params=CONFIGS.keys()) |
| 32 | def hermes_parser(request, qwen_tokenizer: TokenizerLike) -> ToolParser: |
| 33 | config = CONFIGS[request.param] |
| 34 | return config["tool_parser"](qwen_tokenizer) |
| 35 | |
| 36 | |
| 37 | @pytest.fixture |
| 38 | def any_chat_request() -> ChatCompletionRequest: |
| 39 | return ChatCompletionRequest( |
| 40 | seed=42, |
| 41 | model="Qwen/Qwen3-32B", |
| 42 | messages=[], |
| 43 | ) |
| 44 | |
| 45 | |
| 46 | def test_hermes_parser_streaming_just_forward_text( |
| 47 | qwen_tokenizer: TokenizerLike, |
| 48 | hermes_parser: ToolParser, |
| 49 | any_chat_request: ChatCompletionRequest, |
| 50 | ) -> None: |
| 51 | text = """This is some prior text that has nothing to do with tool calling.""" |
| 52 | tokens = qwen_tokenizer.encode(text) |
| 53 | previous_text = "" |
| 54 | delta_messages = [] |
| 55 | for token in tokens: |
| 56 | delta_text = qwen_tokenizer.decode([token]) |
| 57 | current_text = previous_text + delta_text |
| 58 | delta = hermes_parser.extract_tool_calls_streaming( |
| 59 | previous_text=previous_text, |
| 60 | current_text=current_text, |
| 61 | delta_text=delta_text, |
| 62 | previous_token_ids=[], |
| 63 | current_token_ids=[], |
| 64 | delta_token_ids=[], |
| 65 | request=any_chat_request, |
| 66 | ) |
| 67 | previous_text = current_text |
| 68 | delta_messages.append(delta) |
| 69 | |
| 70 | for delta in delta_messages: |
| 71 | assert delta is not None |
| 72 | assert not delta.tool_calls |
| 73 | |
| 74 | print(delta_messages) |
| 75 | assert "".join([delta.content for delta in delta_messages]) == text |
| 76 | |
| 77 | |
| 78 | def test_hermes_parser_streaming_failure_case_bug_19056( |
| 79 | qwen_tokenizer: TokenizerLike, |
| 80 | hermes_parser: ToolParser, |
| 81 | any_chat_request: ChatCompletionRequest, |
| 82 | ) -> None: |
| 83 | text = """<tool_call> |
| 84 | {"name": "final_answer", "arguments": {"trigger": true}} |
| 85 | </tool_call>""" |
| 86 | tokens = qwen_tokenizer.encode(text) |
| 87 | previous_text = "" |
| 88 | delta_messages = [] |
| 89 | for token in tokens: |
| 90 | text = qwen_tokenizer.decode([token]) |
| 91 | current_text = previous_text + text |
| 92 | delta = hermes_parser.extract_tool_calls_streaming( |
| 93 | previous_text=previous_text, |
| 94 | current_text=current_text, |
| 95 | delta_text=text, |
| 96 | previous_token_ids=[], |
| 97 | current_token_ids=[], |
| 98 | delta_token_ids=[], |
| 99 | request=any_chat_request, |
| 100 | ) |
| 101 | previous_text = current_text |
| 102 | if delta is not None: |
| 103 | delta_messages.append(delta) |
| 104 | |
| 105 | assert delta_messages[0].tool_calls[0].function.name == "final_answer" |
| 106 | tool_call_args = "".join( |
| 107 | delta.tool_calls[0].function.arguments or "" for delta in delta_messages |
| 108 | ) |
| 109 | assert tool_call_args == '{"trigger": true}' |
| 110 | |
| 111 | |
| 112 | def test_hermes_parser_streaming( |
| 113 | qwen_tokenizer: TokenizerLike, |
| 114 | hermes_parser: ToolParser, |
| 115 | any_chat_request: ChatCompletionRequest, |
| 116 | ) -> None: |
| 117 | text = '<tool_call>\ |
| 118 | {"name": "get_current_temperature",\ |
| 119 | "arguments": {"location":\ |
| 120 | "San Francisco, California, United States", "unit": "celsius"}}\ |
| 121 | </tool_call>' |
| 122 | |
| 123 | tokens = qwen_tokenizer.encode(text) |
| 124 | previous_text = "" |
| 125 | delta_messages = [] |
| 126 | for token in tokens: |
| 127 | text = qwen_tokenizer.decode([token]) |
| 128 | current_text = previous_text + text |
| 129 | delta = hermes_parser.extract_tool_calls_streaming( |
| 130 | previous_text=previous_text, |
| 131 | current_text=current_text, |
| 132 | delta_text=text, |
| 133 | previous_token_ids=[], |
| 134 | current_token_ids=[], |
| 135 | delta_token_ids=[], |
| 136 | request=any_chat_request, |
| 137 | ) |
| 138 | previous_text = current_text |
| 139 | if delta is not None: |
| 140 | delta_messages.append(delta) |
| 141 | print(delta_messages) |
| 142 | assert delta_messages[0].tool_calls[0].function.name == "get_current_temperature" |
| 143 | # load to normalize whitespace |
| 144 | tool_call_args = json.loads( |
| 145 | "".join( |
| 146 | delta.tool_calls[0].function.arguments or "" for delta in delta_messages |
| 147 | ) |
| 148 | ) |
| 149 | assert tool_call_args == { |
| 150 | "location": "San Francisco, California, United States", |
| 151 | "unit": "celsius", |
| 152 | } |
| 153 | |
| 154 | |
| 155 | def _simulate_streaming( |
| 156 | tokenizer: TokenizerLike, |
| 157 | parser: ToolParser, |
| 158 | request: ChatCompletionRequest, |
| 159 | text: str, |
| 160 | stream_interval: int = 1, |
| 161 | ) -> list: |
| 162 | """Simulate streaming with a given stream_interval. |
| 163 | |
| 164 | Tokens are batched into chunks of `stream_interval` tokens, |
| 165 | mimicking how the output processor delivers them. |
| 166 | Returns a list of non-None DeltaMessages. |
| 167 | """ |
| 168 | tokens = tokenizer.encode(text) |
| 169 | previous_text = "" |
| 170 | delta_messages = [] |
| 171 | for i in range(0, len(tokens), stream_interval): |
| 172 | chunk_ids = tokens[i : i + stream_interval] |
| 173 | delta_text = tokenizer.decode(chunk_ids) |
| 174 | current_text = previous_text + delta_text |
| 175 | delta = parser.extract_tool_calls_streaming( |
| 176 | previous_text=previous_text, |
| 177 | current_text=current_text, |
| 178 | delta_text=delta_text, |
| 179 | previous_token_ids=[], |
| 180 | current_token_ids=[], |
| 181 | delta_token_ids=chunk_ids, |
| 182 | request=request, |
| 183 | ) |
| 184 | previous_text = current_text |
| 185 | if delta is not None: |
| 186 | delta_messages.append(delta) |
| 187 | return delta_messages |
| 188 | |
| 189 | |
| 190 | @pytest.mark.parametrize("stream_interval", [2, 3, 5, 8]) |
| 191 | def test_hermes_streaming_tool_call_with_stream_interval( |
| 192 | qwen_tokenizer: TokenizerLike, |
| 193 | any_chat_request: ChatCompletionRequest, |
| 194 | stream_interval: int, |
| 195 | ) -> None: |
| 196 | """Tool call streaming must produce correct name + args at any interval.""" |
| 197 | text = ( |
| 198 | '<tool_call>{"name": "get_current_temperature", ' |
| 199 | '"arguments": {"location": "San Francisco", "unit": "celsius"}}' |
| 200 | "</tool_call>" |
| 201 | ) |
| 202 | parser = Hermes2ProToolParser(qwen_tokenizer) |
| 203 | deltas = _simulate_streaming( |
| 204 | qwen_tokenizer, parser, any_chat_request, text, stream_interval |
| 205 | ) |
| 206 | |
| 207 | # Flatten all DeltaToolCalls across all deltas. |
| 208 | tool_deltas = [tc for d in deltas if d.tool_calls for tc in d.tool_calls] |
| 209 | assert tool_deltas, "Expected at least one tool call delta" |
| 210 | assert tool_deltas[0].function.name == "get_current_temperature" |
| 211 | |
| 212 | # Concatenated arguments must be valid JSON matching the original. |
| 213 | args_str = "".join(tc.function.arguments or "" for tc in tool_deltas) |
| 214 | assert json.loads(args_str) == { |
| 215 | "location": "San Francisco", |
| 216 | "unit": "celsius", |
| 217 | } |
| 218 | |
| 219 | |
| 220 | @pytest.mark.parametrize("stream_interval", [2, 3, 5, 8]) |
| 221 | def test_hermes_streaming_content_then_tool_call_with_stream_interval( |
| 222 | qwen_tokenizer: TokenizerLike, |
| 223 | any_chat_request: ChatCompletionRequest, |
| 224 | stream_interval: int, |
| 225 | ) -> None: |
| 226 | """Content before a tool call must be fully streamed, then tool call.""" |
| 227 | text = ( |
| 228 | "Sure, let me check the weather." |
| 229 | '<tool_call>{"name": "get_weather", ' |
| 230 | '"arguments": {"city": "NYC"}}</tool_call>' |
| 231 | ) |
| 232 | parser = Hermes2ProToolParser(qwen_tokenizer) |
| 233 | deltas = _simulate_streaming( |
| 234 | qwen_tokenizer, parser, any_chat_request, text, stream_interval |
| 235 | ) |
| 236 | |
| 237 | content_deltas = [d for d in deltas if d.content] |
| 238 | tool_deltas = [d for d in deltas if d.tool_calls] |
| 239 | |
| 240 | # Content must reconstruct the prefix. |
| 241 | content_str = "".join(d.content for d in content_deltas) |
| 242 | assert content_str == "Sure, let me check the weather." |
| 243 | |
| 244 | # Tool call must be correct. |
| 245 | tool_calls = [tc for d in tool_deltas for tc in d.tool_calls] |
| 246 | assert tool_calls[0].function.name == "get_weather" |
| 247 | args_str = "".join(tc.function.arguments or "" for tc in tool_calls) |
| 248 | assert json.loads(args_str) == {"city": "NYC"} |
| 249 | |
| 250 | |
| 251 | @pytest.mark.parametrize("stream_interval", [1, 2, 4]) |
| 252 | def test_hermes_streaming_multiple_tool_calls_with_stream_interval( |
| 253 | qwen_tokenizer: TokenizerLike, |
| 254 | any_chat_request: ChatCompletionRequest, |
| 255 | stream_interval: int, |
| 256 | ) -> None: |
| 257 | """Multiple sequential tool calls must each be streamed correctly.""" |
| 258 | text = ( |
| 259 | '<tool_call>{"name": "search", "arguments": {"q": "cats"}}</tool_call>' |
| 260 | '<tool_call>{"name": "search", "arguments": {"q": "dogs"}}</tool_call>' |
| 261 | ) |
| 262 | parser = Hermes2ProToolParser(qwen_tokenizer) |
| 263 | deltas = _simulate_streaming( |
| 264 | qwen_tokenizer, parser, any_chat_request, text, stream_interval |
| 265 | ) |
| 266 | |
| 267 | # Flatten all DeltaToolCalls across all deltas. |
| 268 | all_tool_calls = [tc for d in deltas if d.tool_calls for tc in d.tool_calls] |
| 269 | |
| 270 | # Separate by tool index. |
| 271 | tool0 = [tc for tc in all_tool_calls if tc.index == 0] |
| 272 | tool1 = [tc for tc in all_tool_calls if tc.index == 1] |
| 273 | |
| 274 | assert tool0[0].function.name == "search" |
| 275 | args0 = "".join(tc.function.arguments or "" for tc in tool0) |
| 276 | assert json.loads(args0) == {"q": "cats"} |
| 277 | |
| 278 | assert tool1[0].function.name == "search" |
| 279 | args1 = "".join(tc.function.arguments or "" for tc in tool1) |
| 280 | assert json.loads(args1) == {"q": "dogs"} |
| 281 | |
| 282 | |
| 283 | @pytest.mark.parametrize("stream_interval", [2, 5]) |
| 284 | def test_hermes_streaming_boolean_args_with_stream_interval( |
| 285 | qwen_tokenizer: TokenizerLike, |
| 286 | any_chat_request: ChatCompletionRequest, |
| 287 | stream_interval: int, |
| 288 | ) -> None: |
| 289 | """Regression test for bug #19056 with stream_interval > 1.""" |
| 290 | text = ( |
| 291 | "<tool_call>\n" |
| 292 | '{"name": "final_answer", "arguments": {"trigger": true}}\n' |
| 293 | "</tool_call>" |
| 294 | ) |
| 295 | parser = Hermes2ProToolParser(qwen_tokenizer) |
| 296 | deltas = _simulate_streaming( |
| 297 | qwen_tokenizer, parser, any_chat_request, text, stream_interval |
| 298 | ) |
| 299 | |
| 300 | tool_calls = [tc for d in deltas if d.tool_calls for tc in d.tool_calls] |
| 301 | assert tool_calls[0].function.name == "final_answer" |
| 302 | args_str = "".join(tc.function.arguments or "" for tc in tool_calls) |
| 303 | assert json.loads(args_str) == {"trigger": True} |
| 304 | |
| 305 | |
| 306 | @pytest.mark.parametrize("stream_interval", [2, 3, 5]) |
| 307 | def test_hermes_streaming_just_forward_text_with_stream_interval( |
| 308 | qwen_tokenizer: TokenizerLike, |
| 309 | any_chat_request: ChatCompletionRequest, |
| 310 | stream_interval: int, |
| 311 | ) -> None: |
| 312 | """Plain text with no tool calls must be fully forwarded.""" |
| 313 | text = "This is plain text with no tool calling involved." |
| 314 | parser = Hermes2ProToolParser(qwen_tokenizer) |
| 315 | deltas = _simulate_streaming( |
| 316 | qwen_tokenizer, parser, any_chat_request, text, stream_interval |
| 317 | ) |
| 318 | |
| 319 | for d in deltas: |
| 320 | assert not d.tool_calls |
| 321 | assert "".join(d.content for d in deltas) == text |
| 322 | |
| 323 | |
| 324 | def test_hermes_parser_non_streaming_no_tool_call( |
| 325 | hermes_parser: ToolParser, |
| 326 | any_chat_request: ChatCompletionRequest, |
| 327 | ) -> None: |
| 328 | text = """This is not a tool call.""" |
| 329 | tool_call = hermes_parser.extract_tool_calls( |
| 330 | model_output=text, |
| 331 | request=any_chat_request, |
| 332 | ) |
| 333 | |
| 334 | assert tool_call is not None |
| 335 | assert not tool_call.tools_called |
| 336 | |
| 337 | |
| 338 | def test_hermes_parser_non_streaming_tool_call_between_tags( |
| 339 | hermes_parser: ToolParser, |
| 340 | any_chat_request: ChatCompletionRequest, |
| 341 | ) -> None: |
| 342 | text = """<tool_call> |
| 343 | {"name": "final_answer", "arguments": {"trigger": true}} |
| 344 | </tool_call>""" |
| 345 | tool_call = hermes_parser.extract_tool_calls( |
| 346 | model_output=text, |
| 347 | request=any_chat_request, |
| 348 | ) |
| 349 | |
| 350 | assert tool_call is not None |
| 351 | assert tool_call.tools_called |
| 352 | assert tool_call.tool_calls[0].function.name == "final_answer" |
| 353 | assert tool_call.tool_calls[0].function.arguments == '{"trigger": true}' |
| 354 | |
| 355 | |
| 356 | def test_hermes_parser_non_streaming_tool_call_until_eos( |
| 357 | hermes_parser: ToolParser, |
| 358 | any_chat_request: ChatCompletionRequest, |
| 359 | ) -> None: |
| 360 | if isinstance(hermes_parser, Granite4ToolParser): |
| 361 | pytest.skip(reason="The Granite4 tool parser enforces a complete response") |
| 362 | |
| 363 | text = """<tool_call> |
| 364 | {"name": "final_answer", "arguments": {"trigger": true}}""" |
| 365 | tool_call = hermes_parser.extract_tool_calls( |
| 366 | model_output=text, |
| 367 | request=any_chat_request, |
| 368 | ) |
| 369 | |
| 370 | assert tool_call is not None |
| 371 | assert tool_call.tools_called |
| 372 | assert tool_call.tool_calls[0].function.name == "final_answer" |
| 373 | assert tool_call.tool_calls[0].function.arguments == '{"trigger": true}' |
| 374 | |
| 375 | |
| 376 | def test_hermes_parser_non_streaming_tool_call_invalid_json( |
| 377 | hermes_parser: ToolParser, |
| 378 | any_chat_request: ChatCompletionRequest, |
| 379 | ) -> None: |
| 380 | # Missing closing brace to trigger exception |
| 381 | text = """<tool_call> |
| 382 | {"name": "final_answer", "arguments": {"trigger": true}""" |
| 383 | tool_call = hermes_parser.extract_tool_calls( |
| 384 | model_output=text, |
| 385 | request=any_chat_request, |
| 386 | ) |
| 387 | |
| 388 | assert tool_call is not None |
| 389 | assert not tool_call.tools_called |
| 390 | |
| 391 | |
| 392 | def test_hermes_streaming_content_and_tool_call_in_single_chunk( |
| 393 | qwen_tokenizer: TokenizerLike, |
| 394 | any_chat_request: ChatCompletionRequest, |
| 395 | ) -> None: |
| 396 | """Content + complete tool call in one chunk must both be emitted.""" |
| 397 | text = 'Hi!<tool_call>{"name": "f", "arguments": {"x": 1}}</tool_call>' |
| 398 | # Use a stream_interval large enough to guarantee a single chunk. |
| 399 | parser = Hermes2ProToolParser(qwen_tokenizer) |
| 400 | deltas = _simulate_streaming( |
| 401 | qwen_tokenizer, |
| 402 | parser, |
| 403 | any_chat_request, |
| 404 | text, |
| 405 | stream_interval=9999, |
| 406 | ) |
| 407 | |
| 408 | content_parts = [d.content for d in deltas if d.content] |
| 409 | tool_parts = [tc for d in deltas if d.tool_calls for tc in d.tool_calls] |
| 410 | |
| 411 | assert "".join(content_parts) == "Hi!" |
| 412 | assert tool_parts[0].function.name == "f" |
| 413 | args_str = "".join(tc.function.arguments or "" for tc in tool_parts) |
| 414 | assert json.loads(args_str) == {"x": 1} |