@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e06f504a76
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Supports tokens and arrays of tokens as inputs to the OpenAI completion API (#715)

WanMok committed Aug 11, 2023 at 12:14 UTC e06f504a761aba85ba34472d6c2544b626c311a8
2 files changed +45 -16
vllm/entrypoints/openai/api_server.py
+43 -15
@@ -3,18 +3,18 @@
3
4 import argparse
5 import asyncio
6 -from http import HTTPStatus
6 import json
7 import time
9 -from typing import AsyncGenerator, Dict, List, Optional
10 -from packaging import version
8 +from http import HTTPStatus
9 +from typing import AsyncGenerator, Dict, List, Optional, Tuple, Union
10
11 import fastapi
12 +import uvicorn
13 from fastapi import BackgroundTasks, Request
14 from fastapi.exceptions import RequestValidationError
15 from fastapi.middleware.cors import CORSMiddleware
16 from fastapi.responses import JSONResponse, StreamingResponse
17 -import uvicorn
17 +from packaging import version
18
19 from vllm.engine.arg_utils import AsyncEngineArgs
20 from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -115,8 +115,18 @@ async def get_gen_prompt(request) -> str:
115 return prompt
116
117
118 -async def check_length(request, prompt):
119 - input_ids = tokenizer(prompt).input_ids
118 +async def check_length(
119 + request: Union[ChatCompletionRequest, CompletionRequest],
120 + prompt: Optional[str] = None,
121 + prompt_ids: Optional[List[int]] = None
122 +) -> Tuple[List[int], Optional[JSONResponse]]:
123 + assert (not (prompt is None and prompt_ids is None)
124 + and not (prompt is not None and prompt_ids is not None)
125 + ), "Either prompt or prompt_ids should be provided."
126 + if prompt_ids is not None:
127 + input_ids = prompt_ids
128 + else:
129 + input_ids = tokenizer(prompt).input_ids
130 token_num = len(input_ids)
131
132 if token_num + request.max_tokens > max_model_len:
@@ -191,7 +201,7 @@ async def create_chat_completion(raw_request: Request):
201 "logit_bias is not currently supported")
202
203 prompt = await get_gen_prompt(request)
194 - token_ids, error_check_ret = await check_length(request, prompt)
204 + token_ids, error_check_ret = await check_length(request, prompt=prompt)
205 if error_check_ret is not None:
206 return error_check_ret
207
@@ -376,19 +386,31 @@ async def create_completion(raw_request: Request):
386
387 model_name = request.model
388 request_id = f"cmpl-{random_uuid()}"
389 +
390 + use_token_ids = False
391 if isinstance(request.prompt, list):
392 if len(request.prompt) == 0:
393 return create_error_response(HTTPStatus.BAD_REQUEST,
394 "please provide at least one prompt")
383 - if len(request.prompt) > 1:
384 - return create_error_response(
385 - HTTPStatus.BAD_REQUEST,
386 - "multiple prompts in a batch is not currently supported")
387 - prompt = request.prompt[0]
395 + first_element = request.prompt[0]
396 + if isinstance(first_element, int):
397 + use_token_ids = True
398 + prompt = request.prompt
399 + elif isinstance(first_element, (str, list)):
400 + # TODO: handles multiple prompt case in list[list[int]]
401 + if len(request.prompt) > 1:
402 + return create_error_response(
403 + HTTPStatus.BAD_REQUEST,
404 + "multiple prompts in a batch is not currently supported")
405 + use_token_ids = not isinstance(first_element, str)
406 + prompt = request.prompt[0]
407 else:
408 prompt = request.prompt
409
391 - token_ids, error_check_ret = await check_length(request, prompt)
410 + if use_token_ids:
411 + _, error_check_ret = await check_length(request, prompt_ids=prompt)
412 + else:
413 + token_ids, error_check_ret = await check_length(request, prompt=prompt)
414 if error_check_ret is not None:
415 return error_check_ret
416
@@ -411,8 +433,14 @@ async def create_completion(raw_request: Request):
433 except ValueError as e:
434 return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
435
414 - result_generator = engine.generate(prompt, sampling_params, request_id,
415 - token_ids)
436 + if use_token_ids:
437 + result_generator = engine.generate(None,
438 + sampling_params,
439 + request_id,
440 + prompt_token_ids=prompt)
441 + else:
442 + result_generator = engine.generate(prompt, sampling_params, request_id,
443 + token_ids)
444
445 # Similar to the OpenAI API, when n != best_of, we do not stream the
446 # results. In addition, we do not stream the results when use beam search.
vllm/entrypoints/openai/protocol.py
+2 -1
@@ -74,7 +74,8 @@ class ChatCompletionRequest(BaseModel):
74
75 class CompletionRequest(BaseModel):
76 model: str
77 - prompt: Union[str, List[str]]
77 + # a string, array of strings, array of tokens, or array of token arrays
78 + prompt: Union[List[int], List[List[int]], str, List[str]]
79 suffix: Optional[str] = None
80 max_tokens: Optional[int] = 16
81 temperature: Optional[float] = 1.0