@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0ffded812a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[Fix] Better error message for batched prompts (#342)

Zhuohan Li committed Jul 3, 2023 at 09:27 UTC 0ffded812aa9193838c3daae87981b46b2c6068c
1 file changed +7 -1
vllm/entrypoints/openai/api_server.py
+7 -1
@@ -358,7 +358,13 @@ async def create_completion(raw_request: Request):
358 model_name = request.model
359 request_id = f"cmpl-{random_uuid()}"
360 if isinstance(request.prompt, list):
361 - assert len(request.prompt) == 1
361 + if len(request.prompt) == 0:
362 + return create_error_response(HTTPStatus.BAD_REQUEST,
363 + "please provide at least one prompt")
364 + if len(request.prompt) > 1:
365 + return create_error_response(HTTPStatus.BAD_REQUEST,
366 + "multiple prompts in a batch is not "
367 + "currently supported")
368 prompt = request.prompt[0]
369 else:
370 prompt = request.prompt