@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b5f93d0631
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Only fail if logit_bias has actual values (#1045)

Lukas Kreussel committed Sep 15, 2023 at 02:33 UTC b5f93d063126dd94518a7773fed74c5b07823719
1 file changed +2 -2
vllm/entrypoints/openai/api_server.py
+2 -2
@@ -196,7 +196,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
196 if error_check_ret is not None:
197 return error_check_ret
198
199 - if request.logit_bias is not None:
199 + if request.logit_bias is not None and len(request.logit_bias) > 0:
200 # TODO: support logit_bias in vLLM engine.
201 return create_error_response(HTTPStatus.BAD_REQUEST,
202 "logit_bias is not currently supported")
@@ -379,7 +379,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
379 return create_error_response(HTTPStatus.BAD_REQUEST,
380 "suffix is not currently supported")
381
382 - if request.logit_bias is not None:
382 + if request.logit_bias is not None and len(request.logit_bias) > 0:
383 # TODO: support logit_bias in vLLM engine.
384 return create_error_response(HTTPStatus.BAD_REQUEST,
385 "logit_bias is not currently supported")