A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Only fail if logit_bias has actual values (#1045)
Lukas Kreussel committed
Sep 15, 2023 at 02:33 UTC
b5f93d063126dd94518a7773fed74c5b07823719
1 file changed
+2
-2
vllm/entrypoints/openai/api_server.py
+2
-2
@@ -196,7 +196,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
196
if error_check_ret is not None:
197
return error_check_ret
198
199
- if request.logit_bias is not None:
199
+ if request.logit_bias is not None and len(request.logit_bias) > 0:
200
# TODO: support logit_bias in vLLM engine.
201
return create_error_response(HTTPStatus.BAD_REQUEST,
202
"logit_bias is not currently supported")
@@ -379,7 +379,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
379
return create_error_response(HTTPStatus.BAD_REQUEST,
380
"suffix is not currently supported")
381
382
- if request.logit_bias is not None:
382
+ if request.logit_bias is not None and len(request.logit_bias) > 0:
383
# TODO: support logit_bias in vLLM engine.
384
return create_error_response(HTTPStatus.BAD_REQUEST,
385
"logit_bias is not currently supported")