A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Supports tokens and arrays of tokens as inputs to the OpenAI completion API (#715)
WanMok committed
Aug 11, 2023 at 12:14 UTC
e06f504a761aba85ba34472d6c2544b626c311a8
2 files changed
+45
-16
vllm/entrypoints/openai/api_server.py
+43
-15
@@ -3,18 +3,18 @@
3
4
import argparse
5
import asyncio
6
-from http import HTTPStatus
6
import json
7
import time
9
-from typing import AsyncGenerator, Dict, List, Optional
10
-from packaging import version
8
+from http import HTTPStatus
9
+from typing import AsyncGenerator, Dict, List, Optional, Tuple, Union
10
11
import fastapi
12
+import uvicorn
13
from fastapi import BackgroundTasks, Request
14
from fastapi.exceptions import RequestValidationError
15
from fastapi.middleware.cors import CORSMiddleware
16
from fastapi.responses import JSONResponse, StreamingResponse
17
-import uvicorn
17
+from packaging import version
18
19
from vllm.engine.arg_utils import AsyncEngineArgs
20
from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -115,8 +115,18 @@ async def get_gen_prompt(request) -> str:
115
return prompt
116
117
118
-async def check_length(request, prompt):
119
- input_ids = tokenizer(prompt).input_ids
118
+async def check_length(
119
+ request: Union[ChatCompletionRequest, CompletionRequest],
120
+ prompt: Optional[str] = None,
121
+ prompt_ids: Optional[List[int]] = None
122
+) -> Tuple[List[int], Optional[JSONResponse]]:
123
+ assert (not (prompt is None and prompt_ids is None)
124
+ and not (prompt is not None and prompt_ids is not None)
125
+ ), "Either prompt or prompt_ids should be provided."
126
+ if prompt_ids is not None:
127
+ input_ids = prompt_ids
128
+ else:
129
+ input_ids = tokenizer(prompt).input_ids
130
token_num = len(input_ids)
131
132
if token_num + request.max_tokens > max_model_len:
@@ -191,7 +201,7 @@ async def create_chat_completion(raw_request: Request):
201
"logit_bias is not currently supported")
202
203
prompt = await get_gen_prompt(request)
194
- token_ids, error_check_ret = await check_length(request, prompt)
204
+ token_ids, error_check_ret = await check_length(request, prompt=prompt)
205
if error_check_ret is not None:
206
return error_check_ret
207
@@ -376,19 +386,31 @@ async def create_completion(raw_request: Request):
386
387
model_name = request.model
388
request_id = f"cmpl-{random_uuid()}"
389
+
390
+ use_token_ids = False
391
if isinstance(request.prompt, list):
392
if len(request.prompt) == 0:
393
return create_error_response(HTTPStatus.BAD_REQUEST,
394
"please provide at least one prompt")
383
- if len(request.prompt) > 1:
384
- return create_error_response(
385
- HTTPStatus.BAD_REQUEST,
386
- "multiple prompts in a batch is not currently supported")
387
- prompt = request.prompt[0]
395
+ first_element = request.prompt[0]
396
+ if isinstance(first_element, int):
397
+ use_token_ids = True
398
+ prompt = request.prompt
399
+ elif isinstance(first_element, (str, list)):
400
+ # TODO: handles multiple prompt case in list[list[int]]
401
+ if len(request.prompt) > 1:
402
+ return create_error_response(
403
+ HTTPStatus.BAD_REQUEST,
404
+ "multiple prompts in a batch is not currently supported")
405
+ use_token_ids = not isinstance(first_element, str)
406
+ prompt = request.prompt[0]
407
else:
408
prompt = request.prompt
409
391
- token_ids, error_check_ret = await check_length(request, prompt)
410
+ if use_token_ids:
411
+ _, error_check_ret = await check_length(request, prompt_ids=prompt)
412
+ else:
413
+ token_ids, error_check_ret = await check_length(request, prompt=prompt)
414
if error_check_ret is not None:
415
return error_check_ret
416
@@ -411,8 +433,14 @@ async def create_completion(raw_request: Request):
433
except ValueError as e:
434
return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
435
414
- result_generator = engine.generate(prompt, sampling_params, request_id,
415
- token_ids)
436
+ if use_token_ids:
437
+ result_generator = engine.generate(None,
438
+ sampling_params,
439
+ request_id,
440
+ prompt_token_ids=prompt)
441
+ else:
442
+ result_generator = engine.generate(prompt, sampling_params, request_id,
443
+ token_ids)
444
445
# Similar to the OpenAI API, when n != best_of, we do not stream the
446
# results. In addition, we do not stream the results when use beam search.
vllm/entrypoints/openai/protocol.py
+2
-1
@@ -74,7 +74,8 @@ class ChatCompletionRequest(BaseModel):
74
75
class CompletionRequest(BaseModel):
76
model: str
77
- prompt: Union[str, List[str]]
77
+ # a string, array of strings, array of tokens, or array of token arrays
78
+ prompt: Union[List[int], List[List[int]], str, List[str]]
79
suffix: Optional[str] = None
80
max_tokens: Optional[int] = 16
81
temperature: Optional[float] = 1.0