@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5a60699c45
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix]: Fix the logic for deciding if tool parsing is used (#8366)

tomeras91 committed Sep 12, 2024 at 06:55 UTC 5a60699c452c0b9b8086a978d8572c257c2c3cc4
1 file changed +1 -1
vllm/entrypoints/openai/serving_chat.py
+1 -1
@@ -607,7 +607,7 @@ class OpenAIServingChat(OpenAIServing):
607
608 # if auto tools are not enabled, and a named tool choice using
609 # outlines is not being used
610 - if not (self.enable_auto_tools
610 + if (not self.enable_auto_tools
611 or not self.tool_parser) and not isinstance(
612 request.tool_choice,
613 ChatCompletionNamedToolChoiceParam):