@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cb0b443274
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 17:15 UTC

[Misc] Add 20 regression tests for 11 tool parser bug fixes (#38172)

Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>

Ben Browning committed Mar 31, 2026 at 23:00 UTC cb0b4432746f8a7aaeeba9a8068e8fd4baf84a0a
7 files changed +700
tests/tool_parsers/test_deepseekv32_tool_parser.py
+84
@@ -11,6 +11,7 @@ from unittest.mock import MagicMock
11
12 import pytest
13
14 +from tests.tool_parsers.utils import run_tool_extraction_streaming
15 from vllm.entrypoints.openai.chat_completion.protocol import (
16 ChatCompletionToolsParam,
17 FunctionDefinition,
@@ -26,6 +27,7 @@ from vllm.tool_parsers.deepseekv32_tool_parser import DeepSeekV32ToolParser
27 # tokenizer object to be truthy (the parser checks `if not self.model_tokenizer`).
28 MOCK_TOKENIZER = MagicMock()
29 MOCK_TOKENIZER.get_vocab.return_value = {}
30 +MOCK_TOKENIZER.tokenize.return_value = []
31
32
33 def make_parser(tools=None) -> DeepSeekV32ToolParser:
@@ -483,6 +485,88 @@ class TestExtractToolCallsStreaming:
485 assert all(not d.tool_calls for d in deltas)
486
487
488 +class TestDelimiterPreservation:
489 + """Regression: fast detokenization skipping DSML delimiters (PR #33964)."""
490 +
491 + @pytest.fixture
492 + def parser(self):
493 + return make_parser()
494 +
495 + def test_delimiter_preserved_fast_detokenization(self, parser):
496 + """DSML delimiters as literal text must still be detected."""
497 + # Delimiters appear as regular text (fast detokenization scenario).
498 + model_output = (
499 + f"{FC_START}\n"
500 + f'{INV_START}get_weather">\n'
501 + f'{PARAM_START}location" string="true">Tokyo{PARAM_END}\n'
502 + f"{INV_END}\n"
503 + f"{FC_END}"
504 + )
505 +
506 + # Non-streaming: parser must detect the tool call
507 + result = parser.extract_tool_calls(model_output, None)
508 + assert result.tools_called
509 + assert len(result.tool_calls) == 1
510 + assert result.tool_calls[0].function.name == "get_weather"
511 + assert json.loads(result.tool_calls[0].function.arguments) == {
512 + "location": "Tokyo"
513 + }
514 +
515 + assert result.content is None
516 +
517 + # With content prefix
518 + prefixed_output = "Here is the weather: " + model_output
519 + result2 = parser.extract_tool_calls(prefixed_output, None)
520 + assert result2.tools_called
521 + assert result2.content == "Here is the weather: "
522 +
523 + def test_tool_detection_skip_special_tokens_false(self, parser):
524 + """Regression: skip_special_tokens must be False when tools are enabled."""
525 + # adjust_request must set skip_special_tokens=False
526 + tool = make_tool_param(
527 + "search",
528 + {
529 + "type": "object",
530 + "properties": {
531 + "query": {"type": "string"},
532 + },
533 + },
534 + )
535 + request = make_request(tools=[tool])
536 + request.tool_choice = "auto"
537 + adjusted = parser.adjust_request(request)
538 + assert adjusted.skip_special_tokens is False
539 +
540 + full_text = build_tool_call("search", {"query": "vllm documentation"})
541 +
542 + # Non-streaming extraction
543 + non_stream_result = parser.extract_tool_calls(full_text, request)
544 + assert non_stream_result.tools_called
545 + assert len(non_stream_result.tool_calls) == 1
546 + assert non_stream_result.tool_calls[0].function.name == "search"
547 + ns_args = json.loads(non_stream_result.tool_calls[0].function.arguments)
548 + assert ns_args == {"query": "vllm documentation"}
549 +
550 + # Streaming extraction: drive the parser line-by-line
551 + chunks: list[str] = []
552 + remaining = full_text
553 + while remaining:
554 + nl = remaining.find("\n")
555 + if nl == -1:
556 + chunks.append(remaining)
557 + break
558 + chunks.append(remaining[: nl + 1])
559 + remaining = remaining[nl + 1 :]
560 +
561 + reconstructor = run_tool_extraction_streaming(
562 + parser, chunks, request, assert_one_tool_per_delta=False
563 + )
564 + assert len(reconstructor.tool_calls) == 1
565 + assert reconstructor.tool_calls[0].function.name == "search"
566 + streamed_args = json.loads(reconstructor.tool_calls[0].function.arguments)
567 + assert streamed_args == ns_args
568 +
569 +
570 @pytest.fixture(scope="module")
571 def deepseekv32_tokenizer():
572 return get_tokenizer(tokenizer_name="deepseek-ai/DeepSeek-V3.2")
tests/tool_parsers/test_glm4_moe_tool_parser.py
+105
@@ -822,3 +822,108 @@ def test_extract_tool_calls_numeric_deserialization(glm4_moe_tool_parser, mock_r
822 # Boolean should be deserialized as bool
823 assert args["enabled"] is True
824 assert isinstance(args["enabled"], bool)
825 +
826 +
827 +def test_zero_argument_tool_call(glm4_moe_tool_parser, mock_request):
828 + """Regression: zero-argument tool call crash (PR #32321)."""
829 + model_output = """<tool_call>get_time
830 +</tool_call>"""
831 +
832 + extracted = glm4_moe_tool_parser.extract_tool_calls(
833 + model_output, request=mock_request
834 + ) # type: ignore[arg-type]
835 +
836 + assert extracted.tools_called
837 + assert len(extracted.tool_calls) == 1
838 + assert extracted.tool_calls[0].function.name == "get_time"
839 + args = json.loads(extracted.tool_calls[0].function.arguments)
840 + assert args == {}
841 +
842 +
843 +def test_malformed_tool_call_no_regex_match(glm4_moe_tool_parser, mock_request):
844 + """Regression: malformed tool_call with no regex match (PR #32321)."""
845 + model_output = "<tool_call> </tool_call>"
846 +
847 + extracted = glm4_moe_tool_parser.extract_tool_calls(
848 + model_output, request=mock_request
849 + ) # type: ignore[arg-type]
850 +
851 + assert extracted.tools_called is False
852 + assert extracted.tool_calls == []
853 +
854 +
855 +def test_delimiter_preserved_transformers_5x(glm4_moe_tool_parser):
856 + """Regression: adjust_request sets skip_special_tokens=False (PR #31622)."""
857 + # Tools enabled
858 + request_with_tools = ChatCompletionRequest(
859 + model=MODEL,
860 + messages=[],
861 + tools=[
862 + {
863 + "type": "function",
864 + "function": {
865 + "name": "get_weather",
866 + "parameters": {
867 + "type": "object",
868 + "properties": {"city": {"type": "string"}},
869 + },
870 + },
871 + }
872 + ],
873 + ) # type: ignore
874 + adjusted = glm4_moe_tool_parser.adjust_request(request_with_tools)
875 + assert adjusted.skip_special_tokens is False
876 +
877 + # tool_choice="none"
878 + request_no_choice = ChatCompletionRequest(
879 + model=MODEL,
880 + messages=[],
881 + tools=[
882 + {
883 + "type": "function",
884 + "function": {
885 + "name": "get_weather",
886 + "parameters": {
887 + "type": "object",
888 + "properties": {"city": {"type": "string"}},
889 + },
890 + },
891 + }
892 + ],
893 + tool_choice="none",
894 + ) # type: ignore
895 + adjusted_none = glm4_moe_tool_parser.adjust_request(request_no_choice)
896 + assert adjusted_none.skip_special_tokens is True
897 +
898 + # No tools at all
899 + request_no_tools = ChatCompletionRequest(
900 + model=MODEL,
901 + messages=[],
902 + ) # type: ignore
903 + adjusted_empty = glm4_moe_tool_parser.adjust_request(request_no_tools)
904 + assert adjusted_empty.skip_special_tokens is True
905 +
906 +
907 +def test_unicode_characters_preserved(glm4_moe_tool_parser, mock_request):
908 + """Regression: Unicode chars must not be escaped to \\uXXXX (PR #30920)."""
909 + model_output = """<tool_call>send_message
910 +<arg_key>greeting</arg_key>
911 +<arg_value>你好世界</arg_value>
912 +<arg_key>emoji</arg_key>
913 +<arg_value>🎉</arg_value>
914 +</tool_call>"""
915 +
916 + extracted = glm4_moe_tool_parser.extract_tool_calls(
917 + model_output, request=mock_request
918 + ) # type: ignore[arg-type]
919 +
920 + assert extracted.tools_called
921 + assert len(extracted.tool_calls) == 1
922 +
923 + raw_args = extracted.tool_calls[0].function.arguments
924 + assert "你好世界" in raw_args
925 + assert "🎉" in raw_args
926 + assert "\\u4f60" not in raw_args
927 + parsed_args = json.loads(raw_args)
928 + assert parsed_args["greeting"] == "你好世界"
929 + assert parsed_args["emoji"] == "🎉"
tests/tool_parsers/test_kimi_k2_tool_parser.py
+53
@@ -872,6 +872,59 @@ def test_streaming_tool_call_markers_not_leaked(kimi_k2_tool_parser):
872 assert "I'll check the weather." in full_content or len(all_content) > 0
873
874
875 +def test_native_id_extracted_and_placed_on_tool_call(kimi_k2_tool_parser):
876 + """Regression: parser extracts native ID onto ToolCall (PR #32768)."""
877 + model_output = (
878 + "Checking weather. "
879 + "<|tool_calls_section_begin|>"
880 + "<|tool_call_begin|>functions.get_weather:0"
881 + '<|tool_call_argument_begin|>{"city": "Tokyo"}'
882 + "<|tool_call_end|>"
883 + "<|tool_calls_section_end|>"
884 + )
885 +
886 + result = kimi_k2_tool_parser.extract_tool_calls(model_output, request=None)
887 + assert result.tools_called
888 + assert len(result.tool_calls) == 1
889 +
890 + tc = result.tool_calls[0]
891 + # Native ID from model output must be used as the tool call ID
892 + assert tc.id == "functions.get_weather:0"
893 + assert tc.function.name == "get_weather"
894 + assert json.loads(tc.function.arguments) == {"city": "Tokyo"}
895 +
896 +
897 +def test_multi_turn_native_id_continuity(kimi_k2_tool_parser, kimi_k2_tokenizer):
898 + """Regression: native IDs from turn 1 preserved across turns (PR #32768)."""
899 + turn1_output = (
900 + "Let me check. "
901 + "<|tool_calls_section_begin|>"
902 + "<|tool_call_begin|>functions.get_weather:0"
903 + '<|tool_call_argument_begin|>{"city": "Beijing"}'
904 + "<|tool_call_end|>"
905 + "<|tool_calls_section_end|>"
906 + )
907 +
908 + turn1_result = kimi_k2_tool_parser.extract_tool_calls(turn1_output, request=None)
909 + assert turn1_result.tools_called
910 + assert turn1_result.tool_calls[0].id == "functions.get_weather:0"
911 +
912 + # Fresh parser for turn 2
913 + turn2_parser = KimiK2ToolParser(kimi_k2_tokenizer)
914 + turn2_output = (
915 + "Now let me get news. "
916 + "<|tool_calls_section_begin|>"
917 + "<|tool_call_begin|>functions.get_news:0"
918 + '<|tool_call_argument_begin|>{"topic": "weather in Beijing"}'
919 + "<|tool_call_end|>"
920 + "<|tool_calls_section_end|>"
921 + )
922 +
923 + turn2_result = turn2_parser.extract_tool_calls(turn2_output, request=None)
924 + assert turn2_result.tools_called
925 + assert turn2_result.tool_calls[0].id == "functions.get_news:0"
926 +
927 +
928 def test_streaming_multiple_tool_calls_not_leaked(kimi_k2_tool_parser):
929 """
930 Test that MULTIPLE tool calls in streaming mode do not leak into content.
tests/tool_parsers/test_minimax_m2_tool_parser.py
+106
@@ -5,6 +5,10 @@ import json
5
6 import pytest
7
8 +from vllm.entrypoints.openai.chat_completion.protocol import (
9 + ChatCompletionToolsParam,
10 + FunctionDefinition,
11 +)
12 from vllm.tool_parsers.minimax_m2_tool_parser import (
13 MinimaxM2ToolParser,
14 )
@@ -442,3 +446,105 @@ class TestLargeChunks:
446 "city": "Seattle",
447 "days": "5",
448 }
449 +
450 +
451 +class TestAnyOfNullableParam:
452 + """Regression: anyOf nullable parameter parsing (PR #32342)."""
453 +
454 + def test_anyof_nullable_param_non_null_value(self):
455 + """A valid non-null string should be preserved, not collapsed to None."""
456 + tools = [
457 + ChatCompletionToolsParam(
458 + function=FunctionDefinition(
459 + name="update_profile",
460 + parameters={
461 + "type": "object",
462 + "properties": {
463 + "nickname": {
464 + "anyOf": [{"type": "string"}, {"type": "null"}],
465 + },
466 + },
467 + },
468 + ),
469 + )
470 + ]
471 + parser = MinimaxM2ToolParser(FakeTokenizer(), tools=tools)
472 +
473 + results = _feed(
474 + parser,
475 + [
476 + '<minimax:tool_call><invoke name="update_profile">'
477 + '<parameter name="nickname">Alice</parameter>'
478 + "</invoke></minimax:tool_call>",
479 + ],
480 + )
481 + tc = _collect_tool_calls(results)
482 + assert len(tc) == 1
483 + parsed = json.loads(tc[0]["arguments"])
484 + assert parsed["nickname"] == "Alice"
485 +
486 + def test_anyof_nullable_param_null_value(self):
487 + """An actual null-like value should be returned as None/null."""
488 + tools = [
489 + ChatCompletionToolsParam(
490 + function=FunctionDefinition(
491 + name="update_profile",
492 + parameters={
493 + "type": "object",
494 + "properties": {
495 + "nickname": {
496 + "anyOf": [{"type": "string"}, {"type": "null"}],
497 + },
498 + },
499 + },
500 + ),
501 + )
502 + ]
503 + parser = MinimaxM2ToolParser(FakeTokenizer(), tools=tools)
504 +
505 + results = _feed(
506 + parser,
507 + [
508 + '<minimax:tool_call><invoke name="update_profile">'
509 + '<parameter name="nickname">null</parameter>'
510 + "</invoke></minimax:tool_call>",
511 + ],
512 + )
513 + tc = _collect_tool_calls(results)
514 + assert len(tc) == 1
515 + parsed = json.loads(tc[0]["arguments"])
516 + assert parsed["nickname"] is None
517 +
518 + def test_anyof_nullable_param_object_value(self):
519 + """A valid object value in anyOf with null should parse as dict."""
520 + tools = [
521 + ChatCompletionToolsParam(
522 + function=FunctionDefinition(
523 + name="update_settings",
524 + parameters={
525 + "type": "object",
526 + "properties": {
527 + "config": {
528 + "anyOf": [{"type": "object"}, {"type": "null"}],
529 + },
530 + },
531 + },
532 + ),
533 + )
534 + ]
535 + parser = MinimaxM2ToolParser(FakeTokenizer(), tools=tools)
536 +
537 + results = _feed(
538 + parser,
539 + [
540 + '<minimax:tool_call><invoke name="update_settings">'
541 + '<parameter name="config">{"theme": "dark", "fontSize": 14}'
542 + "</parameter>"
543 + "</invoke></minimax:tool_call>",
544 + ],
545 + )
546 + tc = _collect_tool_calls(results)
547 + assert len(tc) == 1
548 + parsed = json.loads(tc[0]["arguments"])
549 + assert parsed["config"] == {"theme": "dark", "fontSize": 14}
550 + assert isinstance(parsed["config"], dict)
tests/tool_parsers/test_mistral_tool_parser.py
+61
@@ -890,3 +890,64 @@ def test_extract_tool_calls_streaming_pre_v11_tokenizer_one_chunk(
890 assert expected_content == ""
891 else:
892 assert delta_message.content == expected_content
893 +
894 +
895 +def test_fast_detokenization_text_detection(mistral_tool_parser):
896 + """Regression: bot_token in text but not token_ids (PR #37209)."""
897 + model_output = '[TOOL_CALLS]add{"a": 1, "b": 2}'
898 + # Token IDs that do NOT contain bot_token_id.
899 + fake_token_ids = list(range(99, 99 + 20))
900 +
901 + # First delta: pure content, no bot token yet
902 + delta_message_before = mistral_tool_parser.extract_tool_calls_streaming(
903 + previous_text="",
904 + current_text="Hello",
905 + delta_text="Hello",
906 + previous_token_ids=[],
907 + current_token_ids=[99],
908 + delta_token_ids=[99],
909 + request=None,
910 + )
911 + assert delta_message_before is not None
912 + assert delta_message_before.content == "Hello"
913 + assert not delta_message_before.tool_calls
914 +
915 + # Second delta: bot token in text but NOT in token_ids
916 + delta_message = mistral_tool_parser.extract_tool_calls_streaming(
917 + previous_text="Hello",
918 + current_text="Hello" + model_output,
919 + delta_text=model_output,
920 + previous_token_ids=[99],
921 + current_token_ids=fake_token_ids,
922 + delta_token_ids=fake_token_ids[1:],
923 + request=None,
924 + )
925 + assert delta_message is not None
926 + assert delta_message.tool_calls is not None
927 + assert len(delta_message.tool_calls) > 0
928 + assert delta_message.tool_calls[0].function is not None
929 + assert delta_message.tool_calls[0].function.name == "add"
930 +
931 +
932 +def test_fast_detokenization_text_detection_pre_v11(
933 + mistral_pre_v11_tool_parser,
934 +):
935 + """Regression: bot_token text detection for pre-v11 tokenizer (PR #37209)."""
936 + model_output = '[TOOL_CALLS] [{"name": "add", "arguments":{"a": 1, "b": 2}}]'
937 +
938 + fake_token_ids = list(range(99, 99 + 30))
939 +
940 + delta_message = mistral_pre_v11_tool_parser.extract_tool_calls_streaming(
941 + previous_text="",
942 + current_text=model_output,
943 + delta_text=model_output,
944 + previous_token_ids=[],
945 + current_token_ids=fake_token_ids,
946 + delta_token_ids=fake_token_ids,
947 + request=None,
948 + )
949 + assert delta_message is not None
950 + assert delta_message.tool_calls is not None
951 + assert len(delta_message.tool_calls) > 0
952 + assert delta_message.tool_calls[0].function is not None
953 + assert delta_message.tool_calls[0].function.name == "add"
tests/tool_parsers/test_qwen3coder_tool_parser.py
+154
@@ -974,3 +974,157 @@ fahrenheit
974 assert args["city"] == "Dallas"
975 assert args["state"] == "TX"
976 assert args["unit"] == "fahrenheit"
977 +
978 +
979 +def test_malformed_xml_no_gt_delimiter(qwen3_tool_parser, sample_tools):
980 + """Regression: malformed XML without '>' must not crash (PR #36774)."""
981 + model_output = (
982 + "<tool_call>\n"
983 + "<function=get_current_weather\n"
984 + "<parameter=city>Dallas</parameter>\n"
985 + "</function>\n"
986 + "</tool_call>"
987 + )
988 +
989 + request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
990 + result = qwen3_tool_parser.extract_tool_calls(model_output, request=request)
991 + assert result is not None
992 + assert isinstance(result.tool_calls, list)
993 + assert all(tc is not None for tc in result.tool_calls)
994 +
995 +
996 +def test_none_tool_calls_filtered(qwen3_tool_parser, sample_tools):
997 + """Regression: None tool calls filtered from output (PR #36774)."""
998 + model_output = (
999 + "<tool_call>\n"
1000 + "<function=bad_func_no_gt\n"
1001 + "</function>\n"
1002 + "</tool_call>\n"
1003 + "<tool_call>\n"
1004 + "<function=get_current_weather>\n"
1005 + "<parameter=city>Dallas</parameter>\n"
1006 + "<parameter=state>TX</parameter>\n"
1007 + "</function>\n"
1008 + "</tool_call>"
1009 + )
1010 +
1011 + request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
1012 + result = qwen3_tool_parser.extract_tool_calls(model_output, request=request)
1013 + assert all(tc is not None for tc in result.tool_calls)
1014 + assert result.tools_called
1015 + assert len(result.tool_calls) == 1
1016 + assert result.tool_calls[0].function.name == "get_current_weather"
1017 + args = json.loads(result.tool_calls[0].function.arguments)
1018 + assert args["city"] == "Dallas"
1019 + assert args["state"] == "TX"
1020 +
1021 +
1022 +def test_anyof_parameter_not_double_encoded(qwen3_tokenizer):
1023 + """Regression: anyOf parameters must not be double-encoded (PR #36032)."""
1024 + tools = [
1025 + ChatCompletionToolsParam(
1026 + type="function",
1027 + function={
1028 + "name": "update_record",
1029 + "parameters": {
1030 + "type": "object",
1031 + "properties": {
1032 + "data": {
1033 + "anyOf": [{"type": "object"}, {"type": "null"}],
1034 + },
1035 + },
1036 + },
1037 + },
1038 + )
1039 + ]
1040 +
1041 + parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools)
1042 +
1043 + model_output = (
1044 + "<tool_call>\n"
1045 + "<function=update_record>\n"
1046 + '<parameter=data>{"key": "value", "count": 42}</parameter>\n'
1047 + "</function>\n"
1048 + "</tool_call>"
1049 + )
1050 +
1051 + request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
1052 + result = parser.extract_tool_calls(model_output, request=request)
1053 +
1054 + assert result.tools_called
1055 + assert len(result.tool_calls) == 1
1056 + args = json.loads(result.tool_calls[0].function.arguments)
1057 + assert isinstance(args["data"], dict)
1058 + assert args["data"] == {"key": "value", "count": 42}
1059 +
1060 +
1061 +def test_streaming_multi_param_single_chunk(
1062 + qwen3_tool_parser, qwen3_tokenizer, sample_tools
1063 +):
1064 + """Regression: speculative decode delivering multiple params at once (PR #35615)."""
1065 + request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
1066 +
1067 + deltas = [
1068 + "<tool_call>",
1069 + "\n<function=get_current_weather>",
1070 + "\n", # triggers json_started -> sends "{"
1071 + # This single delta delivers all three parameters at once
1072 + "<parameter=city>\nDallas\n</parameter>"
1073 + "\n<parameter=state>\nTX\n</parameter>"
1074 + "\n<parameter=unit>\nfahrenheit\n</parameter>",
1075 + "\n</function>",
1076 + "\n</tool_call>",
1077 + ]
1078 +
1079 + from tests.tool_parsers.utils import (
1080 + run_tool_extraction_streaming,
1081 + )
1082 +
1083 + reconstructor = run_tool_extraction_streaming(
1084 + qwen3_tool_parser,
1085 + deltas,
1086 + request,
1087 + assert_one_tool_per_delta=False,
1088 + )
1089 +
1090 + assert len(reconstructor.tool_calls) == 1
1091 + args = json.loads(reconstructor.tool_calls[0].function.arguments)
1092 + assert args["city"] == "Dallas"
1093 + assert args["state"] == "TX"
1094 + assert args["unit"] == "fahrenheit"
1095 +
1096 +
1097 +def test_no_double_serialization_string_args(qwen3_tool_parser):
1098 + """Regression: string arguments must not be double-serialized (PR #35615)."""
1099 + tools = [
1100 + ChatCompletionToolsParam(
1101 + type="function",
1102 + function={
1103 + "name": "greet",
1104 + "parameters": {
1105 + "type": "object",
1106 + "properties": {
1107 + "message": {"type": "string"},
1108 + },
1109 + },
1110 + },
1111 + )
1112 + ]
1113 +
1114 + model_output = (
1115 + "<tool_call>\n"
1116 + "<function=greet>\n"
1117 + "<parameter=message>hello world</parameter>\n"
1118 + "</function>\n"
1119 + "</tool_call>"
1120 + )
1121 +
1122 + request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
1123 + result = qwen3_tool_parser.extract_tool_calls(model_output, request=request)
1124 +
1125 + assert result.tools_called
1126 + assert len(result.tool_calls) == 1
1127 + raw_arguments = result.tool_calls[0].function.arguments
1128 + args = json.loads(raw_arguments)
1129 + assert args["message"] == "hello world"
1130 + assert '\\"hello world\\"' not in raw_arguments
tests/tool_parsers/test_step3p5_tool_parser.py
+137
@@ -1431,3 +1431,140 @@ rectangle
1431 assert "<function=calculate_area>" not in extracted_tool_calls.content, (
1432 "Second tool call should not be in content"
1433 )
1434 +
1435 +
1436 +def _accumulate_tool_states(delta_messages):
1437 + """Accumulate tool call state from a stream of DeltaMessage objects."""
1438 + content = ""
1439 + tool_states = {}
1440 + for delta_message in delta_messages:
1441 + if delta_message.content:
1442 + content += delta_message.content
1443 + if delta_message.tool_calls:
1444 + for tool_call in delta_message.tool_calls:
1445 + idx = tool_call.index
1446 + if idx not in tool_states:
1447 + tool_states[idx] = {
1448 + "id": None,
1449 + "name": None,
1450 + "arguments": "",
1451 + "type": None,
1452 + }
1453 + if tool_call.id:
1454 + tool_states[idx]["id"] = tool_call.id
1455 + if tool_call.type:
1456 + tool_states[idx]["type"] = tool_call.type
1457 + if tool_call.function:
1458 + if tool_call.function.name:
1459 + tool_states[idx]["name"] = tool_call.function.name
1460 + if tool_call.function.arguments is not None:
1461 + tool_states[idx]["arguments"] += tool_call.function.arguments
1462 + return content, tool_states
1463 +
1464 +
1465 +def test_streaming_mtp_variable_chunks(
1466 + step3p5_tool_parser, step3p5_tokenizer, sample_tools
1467 +):
1468 + """Regression: MTP variable-size chunks spanning param boundaries (PR #33690)."""
1469 + request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
1470 +
1471 + delta_text_chunks = [
1472 + "<tool_call>\n<function=get_current_weather>\n<parameter=city>\n",
1473 + "Dallas\n</parameter>\n<parameter=state>\nTX",
1474 + "\n</parameter>\n<parameter=unit>\nfahrenheit\n</parameter>",
1475 + "\n</function>\n</tool_call>",
1476 + ]
1477 +
1478 + _, tool_states = _accumulate_tool_states(
1479 + stream_delta_message_generator_from_chunks(
1480 + step3p5_tool_parser, step3p5_tokenizer, delta_text_chunks, request
1481 + )
1482 + )
1483 +
1484 + assert len(tool_states) == 1
1485 +
1486 + state = tool_states[0]
1487 + assert state["id"] is not None
1488 + assert state["type"] == "function"
1489 + assert state["name"] == "get_current_weather"
1490 +
1491 + args = json.loads(state["arguments"])
1492 + assert args["city"] == "Dallas"
1493 + assert args["state"] == "TX"
1494 + assert args["unit"] == "fahrenheit"
1495 +
1496 +
1497 +def test_streaming_multi_token_per_step(
1498 + step3p5_tool_parser, step3p5_tokenizer, sample_tools
1499 +):
1500 + """Regression: MTP large chunks spanning multiple tool calls (PR #33690)."""
1501 + model_output = """<tool_call>
1502 +<function=get_current_weather>
1503 +<parameter=city>
1504 +Dallas
1505 +</parameter>
1506 +<parameter=state>
1507 +TX
1508 +</parameter>
1509 +<parameter=unit>
1510 +fahrenheit
1511 +</parameter>
1512 +</function>
1513 +</tool_call>
1514 +<tool_call>
1515 +<function=get_current_weather>
1516 +<parameter=city>
1517 +Orlando
1518 +</parameter>
1519 +<parameter=state>
1520 +FL
1521 +</parameter>
1522 +<parameter=unit>
1523 +celsius
1524 +</parameter>
1525 +</function>
1526 +</tool_call>"""
1527 +
1528 + request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
1529 +
1530 + # MTP-style large chunks
1531 + mtp_chunks = [
1532 + (
1533 + "<tool_call>\n<function=get_current_weather>\n"
1534 + "<parameter=city>\nDallas\n</parameter>\n"
1535 + "<parameter=state>\nTX"
1536 + ),
1537 + (
1538 + "\n</parameter>\n<parameter=unit>\nfahrenheit\n</parameter>\n"
1539 + "</function>\n</tool_call>\n"
1540 + "<tool_call>\n<function=get_current_weather>\n"
1541 + "<parameter=city>\nOrlando\n</parameter>\n"
1542 + "<parameter=state>\nFL\n</parameter>\n"
1543 + "<parameter=unit>\ncelsius\n</parameter>\n"
1544 + "</function>\n</tool_call>"
1545 + ),
1546 + ]
1547 +
1548 + _, mtp_tool_states = _accumulate_tool_states(
1549 + stream_delta_message_generator_from_chunks(
1550 + step3p5_tool_parser, step3p5_tokenizer, mtp_chunks, request
1551 + )
1552 + )
1553 +
1554 + # Token-by-token streaming (reference)
1555 + step3p5_tool_parser_ref = Step3p5ToolParser(step3p5_tokenizer)
1556 + _, ref_tool_states = _accumulate_tool_states(
1557 + stream_delta_message_generator(
1558 + step3p5_tool_parser_ref, step3p5_tokenizer, model_output, request
1559 + )
1560 + )
1561 +
1562 + assert len(mtp_tool_states) == 2
1563 + assert len(ref_tool_states) == 2
1564 +
1565 + # MTP results must match reference
1566 + for idx in range(2):
1567 + assert mtp_tool_states[idx]["name"] == ref_tool_states[idx]["name"]
1568 + mtp_args = json.loads(mtp_tool_states[idx]["arguments"])
1569 + ref_args = json.loads(ref_tool_states[idx]["arguments"])
1570 + assert mtp_args == ref_args