@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 150 lines 5.42 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 """Unit tests for tool_calls Iterable → list materialisation.
4
5 Regression tests for https://github.com/vllm-project/vllm/issues/34792.
6
7 Setting VLLM_LOGGING_LEVEL=debug caused tool calling to break for Mistral
8 models because:
9 1. The OpenAI Python SDK types tool_calls as Iterable[...] in
10 ChatCompletionAssistantMessageParam.
11 2. Pydantic v2, when validating from Python objects (not from raw JSON),
12 wraps Iterable fields in a one-shot lazy iterator.
13 3. Debug logging called model_dump_json() which consumed that iterator.
14 4. The Mistral tokenizer then saw empty tool_calls and raised
15 "ValueError: Unexpected tool call id ...".
16 """
17
18 import pytest
19
20 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
21
22
23 def _make_tool_call(tc_id: str, name: str, args: str) -> dict:
24 return {
25 "id": tc_id,
26 "type": "function",
27 "function": {"name": name, "arguments": args},
28 }
29
30
31 def _make_request(messages: list) -> ChatCompletionRequest:
32 return ChatCompletionRequest(
33 model="test-model",
34 messages=messages,
35 )
36
37
38 def test_tool_calls_list_preserved_after_model_dump():
39 """tool_calls in assistant messages must be readable after model_dump_json.
40
41 When the request is built from Python dicts (as in the Anthropic → OpenAI
42 conversion path), Pydantic v2 previously wrapped the Iterable tool_calls
43 in a one-shot iterator. model_dump_json() consumed it, leaving subsequent
44 readers (e.g. the Mistral tokenizer) with an empty sequence.
45 """
46 tool_call = _make_tool_call("call_abc123", "get_weather", '{"city": "Paris"}')
47 messages = [
48 {"role": "user", "content": "What is the weather in Paris?"},
49 {"role": "assistant", "content": None, "tool_calls": [tool_call]},
50 {
51 "role": "tool",
52 "tool_call_id": "call_abc123",
53 "content": '{"temperature": 20}',
54 },
55 ]
56
57 req = _make_request(messages)
58
59 # Simulate debug logging: serialize the model (this was the trigger)
60 _ = req.model_dump_json()
61
62 # The assistant message must still have accessible tool_calls afterwards
63 assistant_msg = req.messages[1]
64 assert isinstance(assistant_msg, dict)
65 tool_calls = assistant_msg.get("tool_calls")
66 assert tool_calls is not None, "tool_calls must not be None after model_dump_json"
67 assert isinstance(tool_calls, list), "tool_calls must be a list"
68 assert len(tool_calls) > 0, "tool_calls must not be empty after model_dump_json"
69
70
71 def test_tool_calls_from_generator_are_materialised():
72 """tool_calls passed as a generator must be converted to list on validation."""
73 tool_call = _make_tool_call("call_gen1", "search", '{"query": "vllm"}')
74
75 def tool_calls_gen():
76 yield tool_call
77
78 messages = [
79 {"role": "user", "content": "Search for vllm"},
80 {
81 "role": "assistant",
82 "content": None,
83 "tool_calls": tool_calls_gen(), # one-shot generator
84 },
85 ]
86
87 req = _make_request(messages)
88 assistant_msg = req.messages[1]
89 assert isinstance(assistant_msg, dict)
90
91 # Iterate twice — must not raise or return empty on second pass
92 tool_calls_first = list(assistant_msg.get("tool_calls", []))
93 tool_calls_second = list(assistant_msg.get("tool_calls", []))
94
95 assert len(tool_calls_first) == 1, "First read must return the tool call"
96 assert len(tool_calls_second) == 1, "Second read must also return the tool call"
97
98
99 def test_tool_calls_list_passthrough():
100 """tool_calls already provided as a list must remain a list."""
101 tool_call = _make_tool_call("call_list1", "calculate", '{"expr": "2+2"}')
102 messages = [
103 {"role": "user", "content": "Calculate 2+2"},
104 {"role": "assistant", "content": None, "tool_calls": [tool_call]},
105 ]
106
107 req = _make_request(messages)
108 assistant_msg = req.messages[1]
109 assert isinstance(assistant_msg, dict)
110 assert isinstance(assistant_msg.get("tool_calls"), list)
111
112
113 def test_messages_without_tool_calls_unaffected():
114 """Messages without tool_calls must be handled correctly."""
115 messages = [
116 {"role": "system", "content": "You are a helpful assistant."},
117 {"role": "user", "content": "Hello!"},
118 {"role": "assistant", "content": "Hi there!"},
119 ]
120
121 req = _make_request(messages)
122 # None of the messages should have tool_calls injected
123 for msg in req.messages:
124 assert isinstance(msg, dict)
125 assert msg.get("tool_calls") is None or msg.get("tool_calls") == []
126
127
128 @pytest.mark.parametrize("num_tool_calls", [1, 3])
129 def test_multiple_tool_calls_materialised(num_tool_calls: int):
130 """Multiple tool calls in a single message are all preserved."""
131 tool_calls = [
132 _make_tool_call(f"call_{i}", f"func_{i}", f'{{"arg": {i}}}')
133 for i in range(num_tool_calls)
134 ]
135 messages = [
136 {"role": "user", "content": "Do things"},
137 {"role": "assistant", "content": None, "tool_calls": iter(tool_calls)},
138 ]
139
140 req = _make_request(messages)
141 assistant_msg = req.messages[1]
142 assert isinstance(assistant_msg, dict)
143
144 result_tool_calls = assistant_msg.get("tool_calls")
145 assert isinstance(result_tool_calls, list)
146 assert len(result_tool_calls) == num_tool_calls
147
148 # Verify after model_dump_json too
149 _ = req.model_dump_json()
150 assert len(assistant_msg.get("tool_calls", [])) == num_tool_calls