@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0bd2a573a5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Allow send list of str for the Prompt on openai demo endpoint /v1/completions (#323)

* allow str or List[str] for prompt * Update vllm/entrypoints/openai/api_server.py Co-authored-by: Zhuohan Li <zhuohan123@gmail.com> --------- Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

Michele Catalano committed Jul 3, 2023 at 18:17 UTC 0bd2a573a5a88b6e6135145c5ef5abe3ac942f83
2 files changed +6 -2
vllm/entrypoints/openai/api_server.py
+5 -1
@@ -357,7 +357,11 @@ async def create_completion(raw_request: Request):
357
358 model_name = request.model
359 request_id = f"cmpl-{random_uuid()}"
360 - prompt = request.prompt
360 + if isinstance(request.prompt, list):
361 + assert len(request.prompt) == 1
362 + prompt = request.prompt[0]
363 + else:
364 + prompt = request.prompt
365 created_time = int(time.time())
366 try:
367 sampling_params = SamplingParams(
vllm/entrypoints/openai/protocol.py
+1 -1
@@ -73,7 +73,7 @@ class ChatCompletionRequest(BaseModel):
73
74 class CompletionRequest(BaseModel):
75 model: str
76 - prompt: str
76 + prompt: Union[str, List[str]]
77 suffix: Optional[str] = None
78 max_tokens: Optional[int] = 16
79 temperature: Optional[float] = 1.0