A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix]: Fix the logic for deciding if tool parsing is used (#8366)
tomeras91 committed
Sep 12, 2024 at 06:55 UTC
5a60699c452c0b9b8086a978d8572c257c2c3cc4
1 file changed
+1
-1
vllm/entrypoints/openai/serving_chat.py
+1
-1
@@ -607,7 +607,7 @@ class OpenAIServingChat(OpenAIServing):
607
608
# if auto tools are not enabled, and a named tool choice using
609
# outlines is not being used
610
- if not (self.enable_auto_tools
610
+ if (not self.enable_auto_tools
611
or not self.tool_parser) and not isinstance(
612
request.tool_choice,
613
ChatCompletionNamedToolChoiceParam):