@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7d8ffb344f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix] Internal Server Error when tool_choice is incorrect. (#10567)

Signed-off-by: Varun Shenoy <varun.vinayak.shenoy@oracle.com>

Varun Vinayak Shenoy committed Nov 22, 2024 at 21:13 UTC 7d8ffb344f3b9a571d94073644b829eb4baa0a65
2 files changed +20 -6
tests/entrypoints/openai/test_chat.py
+14
@@ -829,6 +829,20 @@ async def test_inconsistent_tool_choice_and_tools(client: openai.AsyncOpenAI,
829 "name": "nondefined_function_name"
830 }
831 })
832 + with pytest.raises(openai.BadRequestError):
833 + await client.chat.completions.create(
834 + model=MODEL_NAME,
835 + messages=messages,
836 + max_completion_tokens=1000,
837 + tools=[{
838 + "type": "function",
839 + "function": {
840 + "name": "dummy_function_name",
841 + "description": "This is a dummy function",
842 + "parameters": sample_json_schema
843 + }
844 + }],
845 + tool_choice={})
846
847
848 @pytest.mark.asyncio
vllm/entrypoints/openai/protocol.py
+6 -6
@@ -478,17 +478,17 @@ class ChatCompletionRequest(OpenAIBaseModel):
478 # it matches a valid tool
479 if isinstance(data["tool_choice"], dict):
480 valid_tool = False
481 - specified_function = data["tool_choice"]["function"]
481 + specified_function = data["tool_choice"].get("function")
482 if not specified_function:
483 raise ValueError(
484 - "Incorrectly formatted `tool_choice`. Should be like "
485 - "`{\"type\": \"function\","
484 + "Expected field `function` in `tool_choice`."
485 + " Correct usage: `{\"type\": \"function\","
486 " \"function\": {\"name\": \"my_function\"}}`")
487 - specified_function_name = specified_function["name"]
487 + specified_function_name = specified_function.get("name")
488 if not specified_function_name:
489 raise ValueError(
490 - "Incorrectly formatted `tool_choice`. Should be like "
491 - "`{\"type\": \"function\", "
490 + "Expected field `name` in `function` in `tool_choice`."
491 + "Correct usage: `{\"type\": \"function\", "
492 "\"function\": {\"name\": \"my_function\"}}`")
493 for tool in data["tools"]:
494 if tool["function"]["name"] == specified_function_name: