A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Fix] Better error message for batched prompts (#342)
Zhuohan Li committed
Jul 3, 2023 at 09:27 UTC
0ffded812aa9193838c3daae87981b46b2c6068c
1 file changed
+7
-1
vllm/entrypoints/openai/api_server.py
+7
-1
@@ -358,7 +358,13 @@ async def create_completion(raw_request: Request):
358
model_name = request.model
359
request_id = f"cmpl-{random_uuid()}"
360
if isinstance(request.prompt, list):
361
- assert len(request.prompt) == 1
361
+ if len(request.prompt) == 0:
362
+ return create_error_response(HTTPStatus.BAD_REQUEST,
363
+ "please provide at least one prompt")
364
+ if len(request.prompt) > 1:
365
+ return create_error_response(HTTPStatus.BAD_REQUEST,
366
+ "multiple prompts in a batch is not "
367
+ "currently supported")
368
prompt = request.prompt[0]
369
else:
370
prompt = request.prompt