A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Allow send list of str for the Prompt on openai demo endpoint /v1/completions (#323)
* allow str or List[str] for prompt * Update vllm/entrypoints/openai/api_server.py Co-authored-by: Zhuohan Li <zhuohan123@gmail.com> --------- Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
Michele Catalano committed
Jul 3, 2023 at 18:17 UTC
0bd2a573a5a88b6e6135145c5ef5abe3ac942f83
2 files changed
+6
-2
vllm/entrypoints/openai/api_server.py
+5
-1
@@ -357,7 +357,11 @@ async def create_completion(raw_request: Request):
357
358
model_name = request.model
359
request_id = f"cmpl-{random_uuid()}"
360
- prompt = request.prompt
360
+ if isinstance(request.prompt, list):
361
+ assert len(request.prompt) == 1
362
+ prompt = request.prompt[0]
363
+ else:
364
+ prompt = request.prompt
365
created_time = int(time.time())
366
try:
367
sampling_params = SamplingParams(
vllm/entrypoints/openai/protocol.py
+1
-1
@@ -73,7 +73,7 @@ class ChatCompletionRequest(BaseModel):
73
74
class CompletionRequest(BaseModel):
75
model: str
76
- prompt: str
76
+ prompt: Union[str, List[str]]
77
suffix: Optional[str] = None
78
max_tokens: Optional[int] = 16
79
temperature: Optional[float] = 1.0