@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ee2da3e9ef
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

fix validation: Only set tool_choice `auto` if at least one tool is provided (#8568)

Chirag Jain committed Sep 27, 2024 at 04:53 UTC ee2da3e9efb38add804e2023d47e9f42f38bd638
2 files changed +72 -1
tests/tool_use/test_chat_completion_request_validations.py new
+71
@@ -0,0 +1,71 @@
1 +import pytest
2 +
3 +from vllm.entrypoints.openai.protocol import ChatCompletionRequest
4 +
5 +
6 +def test_chat_completion_request_with_no_tools():
7 + # tools key is not present
8 + request = ChatCompletionRequest.model_validate({
9 + 'messages': [{
10 + 'role': 'user',
11 + 'content': 'Hello'
12 + }],
13 + 'model':
14 + 'facebook/opt-125m',
15 + })
16 + assert request.tool_choice == 'none'
17 +
18 + # tools key is None
19 + request = ChatCompletionRequest.model_validate({
20 + 'messages': [{
21 + 'role': 'user',
22 + 'content': 'Hello'
23 + }],
24 + 'model':
25 + 'facebook/opt-125m',
26 + 'tools':
27 + None
28 + })
29 + assert request.tool_choice == 'none'
30 +
31 + # tools key present but empty
32 + request = ChatCompletionRequest.model_validate({
33 + 'messages': [{
34 + 'role': 'user',
35 + 'content': 'Hello'
36 + }],
37 + 'model':
38 + 'facebook/opt-125m',
39 + 'tools': []
40 + })
41 + assert request.tool_choice == 'none'
42 +
43 +
44 +def test_chat_completion_request_with_tool_choice_but_no_tools():
45 + with pytest.raises(ValueError,
46 + match="When using `tool_choice`, `tools` must be set."):
47 + ChatCompletionRequest.model_validate({
48 + 'messages': [{
49 + 'role': 'user',
50 + 'content': 'Hello'
51 + }],
52 + 'model':
53 + 'facebook/opt-125m',
54 + 'tool_choice':
55 + 'auto'
56 + })
57 +
58 + with pytest.raises(ValueError,
59 + match="When using `tool_choice`, `tools` must be set."):
60 + ChatCompletionRequest.model_validate({
61 + 'messages': [{
62 + 'role': 'user',
63 + 'content': 'Hello'
64 + }],
65 + 'model':
66 + 'facebook/opt-125m',
67 + 'tool_choice':
68 + 'auto',
69 + 'tools':
70 + None
71 + })
vllm/entrypoints/openai/protocol.py
+1 -1
@@ -386,7 +386,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
386
387 # if "tool_choice" is not specified but tools are provided,
388 # default to "auto" tool_choice
389 - if "tool_choice" not in data and "tools" in data:
389 + if "tool_choice" not in data and data.get("tools"):
390 data["tool_choice"] = "auto"
391
392 # if "tool_choice" is specified -- validation