A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix] Internal Server Error when tool_choice is incorrect. (#10567)
Signed-off-by: Varun Shenoy <varun.vinayak.shenoy@oracle.com>
Varun Vinayak Shenoy committed
Nov 22, 2024 at 21:13 UTC
7d8ffb344f3b9a571d94073644b829eb4baa0a65
2 files changed
+20
-6
tests/entrypoints/openai/test_chat.py
+14
@@ -829,6 +829,20 @@ async def test_inconsistent_tool_choice_and_tools(client: openai.AsyncOpenAI,
829
"name": "nondefined_function_name"
830
}
831
})
832
+ with pytest.raises(openai.BadRequestError):
833
+ await client.chat.completions.create(
834
+ model=MODEL_NAME,
835
+ messages=messages,
836
+ max_completion_tokens=1000,
837
+ tools=[{
838
+ "type": "function",
839
+ "function": {
840
+ "name": "dummy_function_name",
841
+ "description": "This is a dummy function",
842
+ "parameters": sample_json_schema
843
+ }
844
+ }],
845
+ tool_choice={})
846
847
848
@pytest.mark.asyncio
vllm/entrypoints/openai/protocol.py
+6
-6
@@ -478,17 +478,17 @@ class ChatCompletionRequest(OpenAIBaseModel):
478
# it matches a valid tool
479
if isinstance(data["tool_choice"], dict):
480
valid_tool = False
481
- specified_function = data["tool_choice"]["function"]
481
+ specified_function = data["tool_choice"].get("function")
482
if not specified_function:
483
raise ValueError(
484
- "Incorrectly formatted `tool_choice`. Should be like "
485
- "`{\"type\": \"function\","
484
+ "Expected field `function` in `tool_choice`."
485
+ " Correct usage: `{\"type\": \"function\","
486
" \"function\": {\"name\": \"my_function\"}}`")
487
- specified_function_name = specified_function["name"]
487
+ specified_function_name = specified_function.get("name")
488
if not specified_function_name:
489
raise ValueError(
490
- "Incorrectly formatted `tool_choice`. Should be like "
491
- "`{\"type\": \"function\", "
490
+ "Expected field `name` in `function` in `tool_choice`."
491
+ "Correct usage: `{\"type\": \"function\", "
492
"\"function\": {\"name\": \"my_function\"}}`")
493
for tool in data["tools"]:
494
if tool["function"]["name"] == specified_function_name: