A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
fix validation: Only set tool_choice `auto` if at least one tool is provided (#8568)
Chirag Jain committed
Sep 27, 2024 at 04:53 UTC
ee2da3e9efb38add804e2023d47e9f42f38bd638
2 files changed
+72
-1
tests/tool_use/test_chat_completion_request_validations.py
new
+71
@@ -0,0 +1,71 @@
1
+import pytest
2
+
3
+from vllm.entrypoints.openai.protocol import ChatCompletionRequest
4
+
5
+
6
+def test_chat_completion_request_with_no_tools():
7
+ # tools key is not present
8
+ request = ChatCompletionRequest.model_validate({
9
+ 'messages': [{
10
+ 'role': 'user',
11
+ 'content': 'Hello'
12
+ }],
13
+ 'model':
14
+ 'facebook/opt-125m',
15
+ })
16
+ assert request.tool_choice == 'none'
17
+
18
+ # tools key is None
19
+ request = ChatCompletionRequest.model_validate({
20
+ 'messages': [{
21
+ 'role': 'user',
22
+ 'content': 'Hello'
23
+ }],
24
+ 'model':
25
+ 'facebook/opt-125m',
26
+ 'tools':
27
+ None
28
+ })
29
+ assert request.tool_choice == 'none'
30
+
31
+ # tools key present but empty
32
+ request = ChatCompletionRequest.model_validate({
33
+ 'messages': [{
34
+ 'role': 'user',
35
+ 'content': 'Hello'
36
+ }],
37
+ 'model':
38
+ 'facebook/opt-125m',
39
+ 'tools': []
40
+ })
41
+ assert request.tool_choice == 'none'
42
+
43
+
44
+def test_chat_completion_request_with_tool_choice_but_no_tools():
45
+ with pytest.raises(ValueError,
46
+ match="When using `tool_choice`, `tools` must be set."):
47
+ ChatCompletionRequest.model_validate({
48
+ 'messages': [{
49
+ 'role': 'user',
50
+ 'content': 'Hello'
51
+ }],
52
+ 'model':
53
+ 'facebook/opt-125m',
54
+ 'tool_choice':
55
+ 'auto'
56
+ })
57
+
58
+ with pytest.raises(ValueError,
59
+ match="When using `tool_choice`, `tools` must be set."):
60
+ ChatCompletionRequest.model_validate({
61
+ 'messages': [{
62
+ 'role': 'user',
63
+ 'content': 'Hello'
64
+ }],
65
+ 'model':
66
+ 'facebook/opt-125m',
67
+ 'tool_choice':
68
+ 'auto',
69
+ 'tools':
70
+ None
71
+ })
vllm/entrypoints/openai/protocol.py
+1
-1
@@ -386,7 +386,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
386
387
# if "tool_choice" is not specified but tools are provided,
388
# default to "auto" tool_choice
389
- if "tool_choice" not in data and "tools" in data:
389
+ if "tool_choice" not in data and data.get("tools"):
390
data["tool_choice"] = "auto"
391
392
# if "tool_choice" is specified -- validation