| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | |
| 5 | import pytest |
| 6 | |
| 7 | from tests.tool_parsers.common_tests import ( |
| 8 | ToolParserTestConfig, |
| 9 | ToolParserTests, |
| 10 | ) |
| 11 | from vllm.tokenizers import TokenizerLike, get_tokenizer |
| 12 | |
| 13 | |
| 14 | class TestStep3ToolParser(ToolParserTests): |
| 15 | @pytest.fixture(scope="class") |
| 16 | def tokenizer(self) -> TokenizerLike: |
| 17 | return get_tokenizer("stepfun-ai/step3") |
| 18 | |
| 19 | @pytest.fixture |
| 20 | def test_config(self) -> ToolParserTestConfig: |
| 21 | return ToolParserTestConfig( |
| 22 | parser_name="step3", |
| 23 | # Test data |
| 24 | no_tool_calls_output="This is a regular response without any tool calls.", |
| 25 | single_tool_call_output=( |
| 26 | "<｜tool_calls_begin｜><｜tool_call_begin｜>" |
| 27 | '<steptml:invoke name="get_weather">' |
| 28 | '<steptml:parameter name="city">Tokyo</steptml:parameter>' |
| 29 | "</steptml:invoke><｜tool_call_end｜><｜tool_calls_end｜>" |
| 30 | ), |
| 31 | parallel_tool_calls_output=( |
| 32 | "<｜tool_calls_begin｜><｜tool_call_begin｜>" |
| 33 | '<steptml:invoke name="get_weather">' |
| 34 | '<steptml:parameter name="city">Tokyo</steptml:parameter>' |
| 35 | "</steptml:invoke><｜tool_call_end｜><｜tool_sep｜>" |
| 36 | '<｜tool_call_begin｜><steptml:invoke name="get_time">' |
| 37 | '<steptml:parameter name="timezone">Asia/Tokyo</steptml:parameter>' |
| 38 | "</steptml:invoke><｜tool_call_end｜><｜tool_calls_end｜>" |
| 39 | ), |
| 40 | various_data_types_output=( |
| 41 | "<｜tool_calls_begin｜><｜tool_call_begin｜>" |
| 42 | '<steptml:invoke name="test_function">' |
| 43 | '<steptml:parameter name="string_field">hello</steptml:parameter>' |
| 44 | '<steptml:parameter name="int_field">42</steptml:parameter>' |
| 45 | '<steptml:parameter name="float_field">3.14</steptml:parameter>' |
| 46 | '<steptml:parameter name="bool_field">true</steptml:parameter>' |
| 47 | '<steptml:parameter name="null_field">null</steptml:parameter>' |
| 48 | '<steptml:parameter name="array_field">' |
| 49 | '["a", "b", "c"]</steptml:parameter>' |
| 50 | '<steptml:parameter name="object_field">' |
| 51 | '{"nested": "value"}</steptml:parameter>' |
| 52 | "</steptml:invoke><｜tool_call_end｜><｜tool_calls_end｜>" |
| 53 | ), |
| 54 | empty_arguments_output=( |
| 55 | "<｜tool_calls_begin｜><｜tool_call_begin｜>" |
| 56 | '<steptml:invoke name="refresh"></steptml:invoke>' |
| 57 | "<｜tool_call_end｜><｜tool_calls_end｜>" |
| 58 | ), |
| 59 | surrounding_text_output=( |
| 60 | "Let me check the weather for you.\n\n" |
| 61 | "<｜tool_calls_begin｜><｜tool_call_begin｜>" |
| 62 | '<steptml:invoke name="get_weather">' |
| 63 | '<steptml:parameter name="city">Tokyo</steptml:parameter>' |
| 64 | "</steptml:invoke><｜tool_call_end｜><｜tool_calls_end｜>\n\n" |
| 65 | "I'll get that information." |
| 66 | ), |
| 67 | escaped_strings_output=( |
| 68 | "<｜tool_calls_begin｜><｜tool_call_begin｜>" |
| 69 | '<steptml:invoke name="test_function">' |
| 70 | '<steptml:parameter name="quoted">He said "hello"</steptml:parameter>' |
| 71 | '<steptml:parameter name="path">C:\\Users\\file.txt</steptml:parameter>' |
| 72 | '<steptml:parameter name="newline">line1\nline2</steptml:parameter>' |
| 73 | "</steptml:invoke><｜tool_call_end｜><｜tool_calls_end｜>" |
| 74 | ), |
| 75 | malformed_input_outputs=[ |
| 76 | ( |
| 77 | "<｜tool_calls_begin｜><｜tool_call_begin｜>" |
| 78 | '<steptml:invoke name="func">' |
| 79 | ), |
| 80 | ( |
| 81 | '<｜tool_call_begin｜><steptml:invoke name="func">' |
| 82 | "</steptml:invoke><｜tool_call_end｜>" |
| 83 | ), |
| 84 | ], |
| 85 | # Expected results |
| 86 | single_tool_call_expected_name="get_weather", |
| 87 | single_tool_call_expected_args={"city": "Tokyo"}, |
| 88 | parallel_tool_calls_count=2, |
| 89 | parallel_tool_calls_names=["get_weather", "get_time"], |
| 90 | # xfail markers |
| 91 | xfail_nonstreaming={ |
| 92 | "test_single_tool_call_simple_args": ( |
| 93 | "Step3 parser non-streaming has bugs" |
| 94 | ), |
| 95 | "test_parallel_tool_calls": ("Step3 parser non-streaming has bugs"), |
| 96 | "test_various_data_types": "Step3 parser non-streaming has bugs", |
| 97 | "test_empty_arguments": "Step3 parser non-streaming has bugs", |
| 98 | "test_surrounding_text": "Step3 parser non-streaming has bugs", |
| 99 | "test_escaped_strings": "Step3 parser non-streaming has bugs", |
| 100 | }, |
| 101 | xfail_streaming={ |
| 102 | "test_parallel_tool_calls": ( |
| 103 | "Step3 parser has significant bugs in both streaming " |
| 104 | "and non-streaming" |
| 105 | ), |
| 106 | "test_streaming_reconstruction": ( |
| 107 | "Step3 parser non-streaming has bugs, so streaming " |
| 108 | "doesn't match non-streaming" |
| 109 | ), |
| 110 | }, |
| 111 | supports_typed_arguments=False, |
| 112 | ) |