@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d53cb9cb8e

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 18:30 UTC

[Tool Parser][2/3] Use self.tools instead of request.tools in tool parsers (#38189)

Signed-off-by: sfeng33 <4florafeng@gmail.com>

Flora Feng committed Mar 31, 2026 at 01:41 UTC d53cb9cb8e9eac9d38a3b7bf027df2a6ef15def4

16 files changed +113 -105

tests/tool_parsers/test_deepseekv32_tool_parser.py

+19 -13

@@ -11,6 +11,10 @@ from unittest.mock import MagicMock

12 import pytest

14 +from vllm.entrypoints.openai.chat_completion.protocol import (

15 + ChatCompletionToolsParam,

16 + FunctionDefinition,

17 +)

18 from vllm.tokenizers import get_tokenizer

19 from vllm.tool_parsers.deepseekv32_tool_parser import DeepSeekV32ToolParser

@@ -24,8 +28,8 @@ MOCK_TOKENIZER = MagicMock()

28 MOCK_TOKENIZER.get_vocab.return_value = {}

27 -def make_parser() -> DeepSeekV32ToolParser:

28 - return DeepSeekV32ToolParser(MOCK_TOKENIZER)

31 +def make_parser(tools=None) -> DeepSeekV32ToolParser:

32 + return DeepSeekV32ToolParser(MOCK_TOKENIZER, tools=tools)

35 def make_tool_param(name: str, params: dict) -> MagicMock:

@@ -275,20 +279,22 @@ class TestExtractToolCallsStreaming:

279 content = "".join(d.content for d in deltas if d.content is not None)

280 assert "Thinking" in content

281

278 - def test_type_conversion_in_streaming(self, parser):

279 - tool = make_tool_param(

280 - "add",

281 - {

282 - "type": "object",

283 - "properties": {

284 - "x": {"type": "integer"},

285 - "y": {"type": "integer"},

282 + def test_type_conversion_in_streaming(self):

283 + tool = ChatCompletionToolsParam(

284 + function=FunctionDefinition(

285 + name="add",

286 + parameters={

287 + "type": "object",

288 + "properties": {

289 + "x": {"type": "integer"},

290 + "y": {"type": "integer"},

291 + },

292 },

287 - },

293 + ),

294 )

289 - request = make_request(tools=[tool])

295 + parser = make_parser(tools=[tool])

296 full_text = build_tool_call("add", {"x": "3", "y": "4"})

291 - deltas = self._stream(parser, full_text, request=request)

297 + deltas = self._stream(parser, full_text)

298 args_str = self._reconstruct_args(deltas)

299 assert json.loads(args_str) == {"x": 3, "y": 4}

300

tests/tool_parsers/test_glm47_moe_tool_parser.py

+13 -8

@@ -25,14 +25,8 @@ def glm47_tokenizer():

27 @pytest.fixture

28 -def glm47_tool_parser(glm47_tokenizer):

29 - return Glm47MoeModelToolParser(glm47_tokenizer)

30 -

31 -

32 -@pytest.fixture

33 -def mock_request() -> ChatCompletionRequest:

34 - request = Mock(spec=ChatCompletionRequest)

35 - request.tools = [

28 +def sample_tools():

29 + return [

30 ChatCompletionToolsParam(

31 function=FunctionDefinition(name="get_current_date", parameters={}),

32 ),

@@ -49,6 +43,17 @@ def mock_request() -> ChatCompletionRequest:

43 ),

44 ),

45 ]

46 +

47 +

48 +@pytest.fixture

49 +def glm47_tool_parser(glm47_tokenizer, sample_tools):

50 + return Glm47MoeModelToolParser(glm47_tokenizer, tools=sample_tools)

51 +

52 +

53 +@pytest.fixture

54 +def mock_request(sample_tools) -> ChatCompletionRequest:

55 + request = Mock(spec=ChatCompletionRequest)

56 + request.tools = sample_tools

57 request.tool_choice = "auto"

58 return request

tests/tool_parsers/test_glm4_moe_tool_parser.py

+32 -27

@@ -27,14 +27,8 @@ def glm4_moe_tokenizer():

29 @pytest.fixture

30 -def glm4_moe_tool_parser(glm4_moe_tokenizer):

31 - return Glm4MoeModelToolParser(glm4_moe_tokenizer)

32 -

33 -

34 -@pytest.fixture

35 -def mock_request() -> ChatCompletionRequest:

36 - request = Mock(spec=ChatCompletionRequest)

37 - request.tools = [ # GLM45 parser needs this attribute to enable tool parsing.

30 +def sample_tools():

31 + return [

32 ChatCompletionToolsParam(

33 function=FunctionDefinition(

34 name="get_weather",

@@ -42,6 +36,17 @@ def mock_request() -> ChatCompletionRequest:

36 ),

37 ),

38 ]

39 +

40 +

41 +@pytest.fixture

42 +def glm4_moe_tool_parser(glm4_moe_tokenizer, sample_tools):

43 + return Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=sample_tools)

44 +

45 +

46 +@pytest.fixture

47 +def mock_request(sample_tools) -> ChatCompletionRequest:

48 + request = Mock(spec=ChatCompletionRequest)

49 + request.tools = sample_tools

50 return request

@@ -671,14 +676,13 @@ def test_streaming_json_escape_in_string(glm4_moe_tool_parser, mock_request):

676 assert '"' in parsed["message"] or "world" in parsed["message"]

677

678

674 -def test_streaming_long_content_incremental(glm4_moe_tool_parser):

679 +def test_streaming_long_content_incremental(glm4_moe_tokenizer):

680 """Test incremental streaming of long content (Issue #32829).

681

682 This is the core fix: for long string values like code (4000+ chars),

683 the parser should stream incrementally rather than buffering until

684 complete. This test verifies we get many fragments, not just 1-3.

685 """

681 - _reset_streaming_state(glm4_moe_tool_parser)

686

687 # Bubble sort example from Issue #32829 - realistic long content

688 bubble_sort_code = '''#!/usr/bin/env python3

@@ -705,27 +709,28 @@ if __name__ == "__main__":

709 sorted_arr = bubble_sort(test_arr.copy())

710 print(f"Sorted: {sorted_arr}")'''

711

708 - # Create a request with tool schema to enable string type detection

712 + # Create tools with schema to enable string type detection

713 # This is required for incremental streaming of string values

714 + tools = [

715 + ChatCompletionToolsParam(

716 + function=FunctionDefinition(

717 + name="write_to_file",

718 + parameters={

719 + "type": "object",

720 + "properties": {

721 + "file_path": {"type": "string"},

722 + "content": {"type": "string"},

723 + },

724 + },

725 + ),

726 + ),

727 + ]

728 + glm4_moe_tool_parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools)

729 request = ChatCompletionRequest(

730 model=MODEL,

731 messages=[],

713 - tools=[

714 - {

715 - "type": "function",

716 - "function": {

717 - "name": "write_to_file",

718 - "parameters": {

719 - "type": "object",

720 - "properties": {

721 - "file_path": {"type": "string"},

722 - "content": {"type": "string"},

723 - },

724 - },

725 - },

726 - }

727 - ],

728 - ) # type: ignore

732 + tools=tools,

733 + )

734

735 # Simulate token-based streaming (special tags as single tokens)

736 chunks = [

tests/tool_parsers/test_qwen3coder_tool_parser.py

+10 -12

@@ -31,13 +31,13 @@ def qwen3_tokenizer():

33 @pytest.fixture

34 -def qwen3_tool_parser(qwen3_tokenizer):

35 - return Qwen3CoderToolParser(qwen3_tokenizer)

34 +def qwen3_tool_parser(qwen3_tokenizer, sample_tools):

35 + return Qwen3CoderToolParser(qwen3_tokenizer, tools=sample_tools)

38 @pytest.fixture

39 -def qwen3_xml_tool_parser(qwen3_tokenizer):

40 - return Qwen3XMLToolParser(qwen3_tokenizer)

39 +def qwen3_xml_tool_parser(qwen3_tokenizer, sample_tools):

40 + return Qwen3XMLToolParser(qwen3_tokenizer, tools=sample_tools)

43 @pytest.fixture(params=["xml"])

@@ -376,7 +376,7 @@ TX

376 assert extracted_tool_calls.tool_calls[0].function.name == "get_current_weather"

377

378

379 -def test_extract_tool_calls_type_conversion(qwen3_tool_parser_parametrized):

379 +def test_extract_tool_calls_type_conversion(qwen3_tokenizer):

380 """Test parameter type conversion based on tool schema"""

381 tools = [

382 ChatCompletionToolsParam(

@@ -417,10 +417,9 @@ hello world

417 </function>

418 </tool_call>"""

419

420 + parser = Qwen3XMLToolParser(qwen3_tokenizer, tools=tools)

421 request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)

421 - extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(

422 - model_output, request=request

423 - )

422 + extracted_tool_calls = parser.extract_tool_calls(model_output, request=request)

423

424 args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)

425 assert args["int_param"] == 42

@@ -859,7 +858,7 @@ TX

858

859

860 def test_extract_tool_calls_complex_type_with_single_quote(

862 - qwen3_tool_parser_parametrized,

861 + qwen3_tokenizer,

862 ):

863 """Test parameter type conversion based on tool schema"""

864 tools = [

@@ -889,10 +888,9 @@ def test_extract_tool_calls_complex_type_with_single_quote(

888 </function>

889 </tool_call>"""

890

891 + parser = Qwen3XMLToolParser(qwen3_tokenizer, tools=tools)

892 request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)

893 - extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(

894 - model_output, request=request

895 - )

893 + extracted_tool_calls = parser.extract_tool_calls(model_output, request=request)

894

895 args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)

896 assert args["obj_param"] == {"key": "value"}

tests/tool_parsers/test_seed_oss_tool_parser.py

+2 -2

@@ -30,8 +30,8 @@ def seed_oss_tokenizer():

32 @pytest.fixture

33 -def seed_oss_tool_parser(seed_oss_tokenizer):

34 - return SeedOssToolParser(seed_oss_tokenizer)

33 +def seed_oss_tool_parser(seed_oss_tokenizer, sample_tools):

34 + return SeedOssToolParser(seed_oss_tokenizer, tools=sample_tools)

37 @pytest.fixture

tests/tool_parsers/test_step3p5_tool_parser.py

+8 -10

@@ -28,8 +28,8 @@ def step3p5_tokenizer():

30 @pytest.fixture

31 -def step3p5_tool_parser(step3p5_tokenizer):

32 - return Step3p5ToolParser(step3p5_tokenizer)

31 +def step3p5_tool_parser(step3p5_tokenizer, sample_tools):

32 + return Step3p5ToolParser(step3p5_tokenizer, tools=sample_tools)

35 @pytest.fixture

@@ -386,7 +386,7 @@ TX

386 assert extracted_tool_calls.tool_calls[0].function.name == "get_current_weather"

387

388

389 -def test_extract_tool_calls_type_conversion(step3p5_tool_parser):

389 +def test_extract_tool_calls_type_conversion(step3p5_tokenizer):

390 """Test parameter type conversion based on tool schema"""

391 tools = [

392 ChatCompletionToolsParam(

@@ -427,10 +427,9 @@ hello world

427 </function>

428 </tool_call>"""

429

430 + parser = Step3p5ToolParser(step3p5_tokenizer, tools=tools)

431 request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)

431 - extracted_tool_calls = step3p5_tool_parser.extract_tool_calls(

432 - model_output, request=request

433 - )

432 + extracted_tool_calls = parser.extract_tool_calls(model_output, request=request)

433

434 args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)

435 assert args["int_param"] == 42

@@ -864,7 +863,7 @@ TX

863 assert parsed_args["state"] == "TX"

864

865

867 -def test_extract_tool_calls_complex_type_with_single_quote(step3p5_tool_parser):

866 +def test_extract_tool_calls_complex_type_with_single_quote(step3p5_tokenizer):

867 """Test parameter type conversion based on tool schema"""

868 tools = [

869 ChatCompletionToolsParam(

@@ -893,10 +892,9 @@ def test_extract_tool_calls_complex_type_with_single_quote(step3p5_tool_parser):

892 </function>

893 </tool_call>"""

894

895 + parser = Step3p5ToolParser(step3p5_tokenizer, tools=tools)

896 request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)

897 - extracted_tool_calls = step3p5_tool_parser.extract_tool_calls(

898 - model_output, request=request

899 - )

897 + extracted_tool_calls = parser.extract_tool_calls(model_output, request=request)

898

899 args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)

900 assert args["obj_param"] == {"key": "value"}

vllm/tool_parsers/abstract_tool_parser.py

+10 -1

@@ -10,9 +10,11 @@ from openai.types.responses import (

10 ResponseFormatTextJSONSchemaConfig,

11 ResponseTextConfig,

12 )

13 +from openai.types.responses.function_tool import FunctionTool

15 from vllm.entrypoints.openai.chat_completion.protocol import (

16 ChatCompletionRequest,

17 + ChatCompletionToolsParam,

18 )

19 from vllm.entrypoints.openai.engine.protocol import (

20 DeltaMessage,

@@ -54,7 +56,14 @@ class ToolParser:

56 self.streamed_args_for_tool: list[str] = []

58 self.model_tokenizer = tokenizer

57 - self.tools = tools

59 + if tools:

60 + self.tools: list[ChatCompletionToolsParam | FunctionTool] = [

61 + tool

62 + for tool in tools

63 + if isinstance(tool, (ChatCompletionToolsParam, FunctionTool))

64 + ]

65 + else:

66 + self.tools = []

68 @cached_property

69 def vocab(self) -> dict[str, int]:

vllm/tool_parsers/deepseekv32_tool_parser.py

+3 -6

@@ -142,12 +142,11 @@ class DeepSeekV32ToolParser(ToolParser):

142 self,

143 function_name: str,

144 param_dict: dict[str, str],

145 - request: ChatCompletionRequest | None,

145 ) -> dict[str, Any]:

146 """Convert raw string param values using the tool schema types."""

147 param_config: dict = {}

149 - if request and request.tools:

150 - for tool in request.tools:

148 + if self.tools:

149 + for tool in self.tools:

150 if (

151 hasattr(tool, "function")

152 and tool.function.name == function_name

@@ -241,9 +240,7 @@ class DeepSeekV32ToolParser(ToolParser):

240 invoke_name, invoke_body = complete_invokes[self.current_tool_index]

241 param_dict = self._parse_invoke_params(invoke_body)

242

244 - converted = self._convert_params_with_schema(

245 - invoke_name, param_dict, request

246 - )

243 + converted = self._convert_params_with_schema(invoke_name, param_dict)

244 args_json = json.dumps(converted, ensure_ascii=False)

245 idx = self.current_tool_index

246 self.current_tool_index += 1

vllm/tool_parsers/glm4_moe_tool_parser.py

+2 -2

@@ -189,7 +189,7 @@ class Glm4MoeModelToolParser(ToolParser):

189 for key, value in pairs:

190 arg_key = key.strip()

191 arg_val = value.strip()

192 - if not self._is_string_type(tc_name, arg_key, request.tools):

192 + if not self._is_string_type(tc_name, arg_key, self.tools):

193 arg_val = self._deserialize(arg_val)

194 logger.debug("arg_key = %s, arg_val = %s", arg_key, arg_val)

195 arg_dct[arg_key] = arg_val

@@ -330,7 +330,7 @@ class Glm4MoeModelToolParser(ToolParser):

330 key = (self._pending_key or "").strip()

331

332 is_string = self._is_string_type(

333 - self._current_tool_name, key, request.tools

333 + self._current_tool_name, key, self.tools

334 )

335

336 if is_string:

vllm/tool_parsers/internlm2_tool_parser.py

+1 -1

@@ -200,7 +200,7 @@ class Internlm2ToolParser(ToolParser):

200 request: ChatCompletionRequest,

201 ) -> ExtractedToolCallInformation:

202 text = model_output

203 - tools = request.tools

203 + tools = self.tools

204 if "<|action_start|><|plugin|>" in text:

205 text, action = text.split("<|action_start|><|plugin|>")

206 action = action.split("<|action_end|>".strip())[0]

vllm/tool_parsers/minimax_m2_tool_parser.py

+2 -4

@@ -308,7 +308,7 @@ class MinimaxM2ToolParser(ToolParser):

308 invoke_str = complete_invokes[self.current_tool_index]

309 tool_call = self._parse_single_invoke(

310 invoke_str,

311 - request.tools if request else None,

311 + self.tools,

312 )

313 if not tool_call:

314 self.current_tool_index += 1

@@ -358,9 +358,7 @@ class MinimaxM2ToolParser(ToolParser):

358 for tool_call_match in self.tool_call_complete_regex.findall(model_output):

359 # Find all invokes within this tool_call

360 for invoke_match in self.invoke_complete_regex.findall(tool_call_match):

361 - tool_call = self._parse_single_invoke(

362 - invoke_match, request.tools if request else None

363 - )

361 + tool_call = self._parse_single_invoke(invoke_match, self.tools)

362 if tool_call:

363 tool_calls.append(tool_call)

364

vllm/tool_parsers/qwen3coder_tool_parser.py

+3 -5

@@ -314,7 +314,7 @@ class Qwen3CoderToolParser(ToolParser):

314 )

315

316 tool_calls = [

317 - self._parse_xml_function_call(function_call_str, request.tools)

317 + self._parse_xml_function_call(function_call_str, self.tools)

318 for function_call_str in function_calls

319 ]

320 # Populate prev_tool_call_arr for serving layer to set finish_reason

@@ -607,7 +607,7 @@ class Qwen3CoderToolParser(ToolParser):

607

608 param_config = self._get_arguments_config(

609 self.current_function_name or "",

610 - self.streaming_request.tools if self.streaming_request else None,

610 + self.tools,

611 )

612

613 converted_value = self._convert_param_value(

@@ -666,9 +666,7 @@ class Qwen3CoderToolParser(ToolParser):

666 try:

667 parsed_tool = self._parse_xml_function_call(

668 func_content,

669 - self.streaming_request.tools

670 - if self.streaming_request

671 - else None,

669 + self.tools,

670 )

671 if parsed_tool and self.current_tool_index < len(

672 self.prev_tool_call_arr

vllm/tool_parsers/qwen3xml_tool_parser.py

+2 -4

@@ -1188,8 +1188,7 @@ class Qwen3XMLToolParser(ToolParser):

1188 # Reset tool call tracking arrays for new extraction

1189 self.prev_tool_call_arr = []

1190 self.streamed_args_for_tool = []

1191 - if request:

1192 - self.parser.set_tools(request.tools)

1191 + self.parser.set_tools(self.tools)

1192 result = self.parser.parse_single_streaming_chunks(model_output)

1193 if not result.tool_calls:

1194 return ExtractedToolCallInformation(

@@ -1260,8 +1259,7 @@ class Qwen3XMLToolParser(ToolParser):

1259 # Reset tool call tracking arrays for new streaming session

1260 self.prev_tool_call_arr = []

1261 self.streamed_args_for_tool = []

1263 - if request:

1264 - self.parser.set_tools(request.tools)

1262 + self.parser.set_tools(self.tools)

1263

1264 # Model sometimes outputs separately causing delta_text to be empty.

1265 # If there were tool_calls before and all current tool_calls have ended,

vllm/tool_parsers/seed_oss_tool_parser.py

+2 -2

@@ -312,7 +312,7 @@ class SeedOssToolParser(ToolParser):

312 )

313

314 tool_calls = [

315 - self._parse_xml_function_call(function_call_str, request.tools)

315 + self._parse_xml_function_call(function_call_str, self.tools)

316 for function_call_str in function_calls

317 ]

318

@@ -566,7 +566,7 @@ class SeedOssToolParser(ToolParser):

566 # Parse to get the complete arguments

567 try:

568 parsed_tool = self._parse_xml_function_call(

569 - func_content, request.tools if request else None

569 + func_content, self.tools

570 )

571 if parsed_tool:

572 # Update existing entry in prev_tool_call_arr with complete arguments

vllm/tool_parsers/step3_tool_parser.py

+2 -4

@@ -82,9 +82,8 @@ class Step3ToolParser(ToolParser):

82 self,

83 func_name: str,

84 params: dict[str, Any],

85 - request: ChatCompletionRequest,

85 ) -> dict[str, Any]:

87 - for tool in request.tools or []:

86 + for tool in self.tools or []:

87 if tool.function.name == func_name:

88 schema = tool.function.parameters or {}

89 properties = schema.get("properties", {})

@@ -234,7 +233,6 @@ class Step3ToolParser(ToolParser):

233 final_args = self._cast_arguments(

234 function_name,

235 tool_call_arr.get("parameters", {}), # type: ignore

237 - request,

236 )

237 if final_args:

238 final_args_json = json.dumps(final_args, ensure_ascii=False)

@@ -291,7 +289,7 @@ class Step3ToolParser(ToolParser):

289 function_name, params_dict = self._parse_steptml_invoke(invoke_part)

290

291 if function_name and params_dict is not None:

294 - params_dict = self._cast_arguments(function_name, params_dict, request)

292 + params_dict = self._cast_arguments(function_name, params_dict)

293 params_str = json.dumps(params_dict, ensure_ascii=False)

294 tool_calls.append(

295 ToolCall(

vllm/tool_parsers/step3p5_tool_parser.py

+2 -4

@@ -1385,8 +1385,7 @@ class Step3p5ToolParser(ToolParser):

1385 # Reset tool call tracking arrays for new extraction

1386 self.prev_tool_call_arr = []

1387 self.streamed_args_for_tool = []

1388 - if request:

1389 - self.parser.set_tools(request.tools)

1388 + self.parser.set_tools(self.tools)

1389 result = self.parser.parse_single_streaming_chunks(model_output)

1390 if not result.tool_calls:

1391 return ExtractedToolCallInformation(

@@ -1457,8 +1456,7 @@ class Step3p5ToolParser(ToolParser):

1456 # Reset tool call tracking arrays for new streaming session

1457 self.prev_tool_call_arr = []

1458 self.streamed_args_for_tool = []

1460 - if request:

1461 - self.parser.set_tools(request.tools)

1459 + self.parser.set_tools(self.tools)

1460

1461 # Model sometimes outputs separately causing delta_text to be empty.

1462 # If there were tool_calls before and all current tool_calls have ended,