A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Frontend][Bugfix] Disallow extra fields in OpenAI API (#4355)
Cyrus Leung committed
Apr 27, 2024 at 13:08 UTC
8947bc3c156963dfc66e7ca1e4c436506ed6a512
8 files changed
+113
-55
requirements-common.txt
+1
@@ -8,6 +8,7 @@ py-cpuinfo
8
transformers >= 4.40.0 # Required for StarCoder2 & Llava, Llama 3.
9
tokenizers >= 0.19.1 # Required for Llama 3.
10
fastapi
11
+openai
12
uvicorn[standard]
13
pydantic >= 2.0 # Required for OpenAI server.
14
prometheus_client >= 0.18.0
requirements-dev.txt
-1
@@ -21,7 +21,6 @@ pytest-rerunfailures
21
pytest-shard
22
httpx
23
einops # required for MPT
24
-openai
24
requests
25
ray
26
peft
tests/entrypoints/test_openai_server.py
+16
@@ -15,6 +15,7 @@ import ray
15
import requests
16
# downloading lora to test lora requests
17
from huggingface_hub import snapshot_download
18
+from openai import BadRequestError
19
20
from vllm.transformers_utils.tokenizer import get_tokenizer
21
@@ -770,6 +771,21 @@ async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
771
assert loaded == {"result": 2}, loaded
772
773
774
+async def test_extra_fields(server, client: openai.AsyncOpenAI):
775
+ with pytest.raises(BadRequestError) as exc_info:
776
+ await client.chat.completions.create(
777
+ model=MODEL_NAME,
778
+ messages=[{
779
+ "role": "system",
780
+ "content": "You are a helpful assistant.",
781
+ "extra_field": "0",
782
+ }], # type: ignore
783
+ temperature=0,
784
+ seed=0)
785
+
786
+ assert "extra_forbidden" in exc_info.value.message
787
+
788
+
789
async def test_guided_grammar(server, client: openai.AsyncOpenAI):
790
simple_sql_grammar = """
791
start: select_statement
vllm/entrypoints/openai/cli_args.py
+2
-2
@@ -9,7 +9,7 @@ import json
9
import ssl
10
11
from vllm.engine.arg_utils import AsyncEngineArgs
12
-from vllm.entrypoints.openai.serving_engine import LoRA
12
+from vllm.entrypoints.openai.serving_engine import LoRAModulePath
13
14
15
class LoRAParserAction(argparse.Action):
@@ -18,7 +18,7 @@ class LoRAParserAction(argparse.Action):
18
lora_list = []
19
for item in values:
20
name, path = item.split('=')
21
- lora_list.append(LoRA(name, path))
21
+ lora_list.append(LoRAModulePath(name, path))
22
setattr(namespace, self.dest, lora_list)
23
24
vllm/entrypoints/openai/protocol.py
+35
-29
@@ -4,14 +4,20 @@ import time
4
from typing import Dict, List, Literal, Optional, Union
5
6
import torch
7
-from pydantic import BaseModel, Field, model_validator
7
+from openai.types.chat import ChatCompletionMessageParam
8
+from pydantic import BaseModel, ConfigDict, Field, model_validator
9
from typing_extensions import Annotated
10
11
from vllm.sampling_params import SamplingParams
12
from vllm.utils import random_uuid
13
14
14
-class ErrorResponse(BaseModel):
15
+class OpenAIBaseModel(BaseModel):
16
+ # OpenAI API does not allow extra fields
17
+ model_config = ConfigDict(extra="forbid")
18
+
19
+
20
+class ErrorResponse(OpenAIBaseModel):
21
object: str = "error"
22
message: str
23
type: str
@@ -19,7 +25,7 @@ class ErrorResponse(BaseModel):
25
code: int
26
27
22
-class ModelPermission(BaseModel):
28
+class ModelPermission(OpenAIBaseModel):
29
id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
30
object: str = "model_permission"
31
created: int = Field(default_factory=lambda: int(time.time()))
@@ -34,7 +40,7 @@ class ModelPermission(BaseModel):
40
is_blocking: bool = False
41
42
37
-class ModelCard(BaseModel):
43
+class ModelCard(OpenAIBaseModel):
44
id: str
45
object: str = "model"
46
created: int = Field(default_factory=lambda: int(time.time()))
@@ -44,26 +50,26 @@ class ModelCard(BaseModel):
50
permission: List[ModelPermission] = Field(default_factory=list)
51
52
47
-class ModelList(BaseModel):
53
+class ModelList(OpenAIBaseModel):
54
object: str = "list"
55
data: List[ModelCard] = Field(default_factory=list)
56
57
52
-class UsageInfo(BaseModel):
58
+class UsageInfo(OpenAIBaseModel):
59
prompt_tokens: int = 0
60
total_tokens: int = 0
61
completion_tokens: Optional[int] = 0
62
63
58
-class ResponseFormat(BaseModel):
64
+class ResponseFormat(OpenAIBaseModel):
65
# type must be "json_object" or "text"
66
type: Literal["text", "json_object"]
67
68
63
-class ChatCompletionRequest(BaseModel):
69
+class ChatCompletionRequest(OpenAIBaseModel):
70
# Ordered by official OpenAI API documentation
71
# https://platform.openai.com/docs/api-reference/chat/create
66
- messages: List[Dict[str, str]]
72
+ messages: List[ChatCompletionMessageParam]
73
model: str
74
frequency_penalty: Optional[float] = 0.0
75
logit_bias: Optional[Dict[str, float]] = None
@@ -204,7 +210,7 @@ class ChatCompletionRequest(BaseModel):
210
return data
211
212
207
-class CompletionRequest(BaseModel):
213
+class CompletionRequest(OpenAIBaseModel):
214
# Ordered by official OpenAI API documentation
215
# https://platform.openai.com/docs/api-reference/completions/create
216
model: str
@@ -343,19 +349,19 @@ class CompletionRequest(BaseModel):
349
return data
350
351
346
-class LogProbs(BaseModel):
352
+class LogProbs(OpenAIBaseModel):
353
text_offset: List[int] = Field(default_factory=list)
354
token_logprobs: List[Optional[float]] = Field(default_factory=list)
355
tokens: List[str] = Field(default_factory=list)
356
top_logprobs: Optional[List[Optional[Dict[str, float]]]] = None
357
358
353
-class CompletionResponseChoice(BaseModel):
359
+class CompletionResponseChoice(OpenAIBaseModel):
360
index: int
361
text: str
362
logprobs: Optional[LogProbs] = None
357
- finish_reason: Optional[Literal["stop", "length"]] = None
358
- stop_reason: Union[None, int, str] = Field(
363
+ finish_reason: Optional[str] = None
364
+ stop_reason: Optional[Union[int, str]] = Field(
365
default=None,
366
description=(
367
"The stop string or token id that caused the completion "
@@ -364,7 +370,7 @@ class CompletionResponseChoice(BaseModel):
370
)
371
372
367
-class CompletionResponse(BaseModel):
373
+class CompletionResponse(OpenAIBaseModel):
374
id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
375
object: str = "text_completion"
376
created: int = Field(default_factory=lambda: int(time.time()))
@@ -373,12 +379,12 @@ class CompletionResponse(BaseModel):
379
usage: UsageInfo
380
381
376
-class CompletionResponseStreamChoice(BaseModel):
382
+class CompletionResponseStreamChoice(OpenAIBaseModel):
383
index: int
384
text: str
385
logprobs: Optional[LogProbs] = None
380
- finish_reason: Optional[Literal["stop", "length"]] = None
381
- stop_reason: Union[None, int, str] = Field(
386
+ finish_reason: Optional[str] = None
387
+ stop_reason: Optional[Union[int, str]] = Field(
388
default=None,
389
description=(
390
"The stop string or token id that caused the completion "
@@ -387,7 +393,7 @@ class CompletionResponseStreamChoice(BaseModel):
393
)
394
395
390
-class CompletionStreamResponse(BaseModel):
396
+class CompletionStreamResponse(OpenAIBaseModel):
397
id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
398
object: str = "text_completion"
399
created: int = Field(default_factory=lambda: int(time.time()))
@@ -396,20 +402,20 @@ class CompletionStreamResponse(BaseModel):
402
usage: Optional[UsageInfo] = Field(default=None)
403
404
399
-class ChatMessage(BaseModel):
405
+class ChatMessage(OpenAIBaseModel):
406
role: str
407
content: str
408
409
404
-class ChatCompletionResponseChoice(BaseModel):
410
+class ChatCompletionResponseChoice(OpenAIBaseModel):
411
index: int
412
message: ChatMessage
413
logprobs: Optional[LogProbs] = None
408
- finish_reason: Optional[Literal["stop", "length"]] = None
409
- stop_reason: Union[None, int, str] = None
414
+ finish_reason: Optional[str] = None
415
+ stop_reason: Optional[Union[int, str]] = None
416
417
412
-class ChatCompletionResponse(BaseModel):
418
+class ChatCompletionResponse(OpenAIBaseModel):
419
id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
420
object: str = "chat.completion"
421
created: int = Field(default_factory=lambda: int(time.time()))
@@ -418,20 +424,20 @@ class ChatCompletionResponse(BaseModel):
424
usage: UsageInfo
425
426
421
-class DeltaMessage(BaseModel):
427
+class DeltaMessage(OpenAIBaseModel):
428
role: Optional[str] = None
429
content: Optional[str] = None
430
431
426
-class ChatCompletionResponseStreamChoice(BaseModel):
432
+class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
433
index: int
434
delta: DeltaMessage
435
logprobs: Optional[LogProbs] = None
430
- finish_reason: Optional[Literal["stop", "length"]] = None
431
- stop_reason: Union[None, int, str] = None
436
+ finish_reason: Optional[str] = None
437
+ stop_reason: Optional[Union[int, str]] = None
438
439
434
-class ChatCompletionStreamResponse(BaseModel):
440
+class ChatCompletionStreamResponse(OpenAIBaseModel):
441
id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
442
object: str = "chat.completion.chunk"
443
created: int = Field(default_factory=lambda: int(time.time()))
vllm/entrypoints/openai/serving_chat.py
+44
-11
@@ -1,8 +1,11 @@
1
import codecs
2
import time
3
-from typing import AsyncGenerator, AsyncIterator, List, Optional, Union
3
+from typing import (AsyncGenerator, AsyncIterator, Awaitable, Iterable, List,
4
+ Optional, Tuple, TypedDict, Union, final)
5
6
from fastapi import Request
7
+from openai.types.chat import (ChatCompletionContentPartParam,
8
+ ChatCompletionRole)
9
10
from vllm.engine.async_llm_engine import AsyncLLMEngine
11
from vllm.entrypoints.openai.protocol import (
@@ -10,7 +13,8 @@ from vllm.entrypoints.openai.protocol import (
13
ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
14
ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
15
UsageInfo)
13
-from vllm.entrypoints.openai.serving_engine import LoRA, OpenAIServing
16
+from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
17
+ OpenAIServing)
18
from vllm.logger import init_logger
19
from vllm.model_executor.guided_decoding import (
20
get_guided_decoding_logits_processor)
@@ -20,20 +24,41 @@ from vllm.utils import random_uuid
24
logger = init_logger(__name__)
25
26
27
+@final # So that it should be compatible with Dict[str, str]
28
+class ConversationMessage(TypedDict):
29
+ role: str
30
+ content: str
31
+
32
+
33
class OpenAIServingChat(OpenAIServing):
34
35
def __init__(self,
36
engine: AsyncLLMEngine,
37
served_model_names: List[str],
38
response_role: str,
29
- lora_modules: Optional[List[LoRA]] = None,
30
- chat_template=None):
39
+ lora_modules: Optional[List[LoRAModulePath]] = None,
40
+ chat_template: Optional[str] = None):
41
super().__init__(engine=engine,
42
served_model_names=served_model_names,
43
lora_modules=lora_modules)
44
self.response_role = response_role
45
self._load_chat_template(chat_template)
46
47
+ def _parse_chat_message_content(
48
+ self,
49
+ role: ChatCompletionRole,
50
+ content: Optional[Union[str,
51
+ Iterable[ChatCompletionContentPartParam]]],
52
+ ) -> Tuple[List[ConversationMessage], List[Awaitable[object]]]:
53
+ if content is None:
54
+ return [], []
55
+ if isinstance(content, str):
56
+ return [ConversationMessage(role=role, content=content)], []
57
+
58
+ # To be implemented: https://github.com/vllm-project/vllm/pull/3467
59
+ # To be implemented: https://github.com/vllm-project/vllm/pull/4200
60
+ raise NotImplementedError("Complex input not supported yet")
61
+
62
async def create_chat_completion(
63
self, request: ChatCompletionRequest, raw_request: Request
64
) -> Union[ErrorResponse, AsyncGenerator[str, None],
@@ -52,10 +77,19 @@ class OpenAIServingChat(OpenAIServing):
77
return error_check_ret
78
79
try:
80
+ conversation: List[ConversationMessage] = []
81
+
82
+ for m in request.messages:
83
+ messages, _ = self._parse_chat_message_content(
84
+ m["role"], m["content"])
85
+
86
+ conversation.extend(messages)
87
+
88
prompt = self.tokenizer.apply_chat_template(
56
- conversation=request.messages,
89
+ conversation=conversation,
90
tokenize=False,
58
- add_generation_prompt=request.add_generation_prompt)
91
+ add_generation_prompt=request.add_generation_prompt,
92
+ )
93
except Exception as e:
94
logger.error("Error in applying chat template from request: %s", e)
95
return self.create_error_response(str(e))
@@ -105,9 +139,8 @@ class OpenAIServingChat(OpenAIServing):
139
140
async def chat_completion_stream_generator(
141
self, request: ChatCompletionRequest,
108
- result_generator: AsyncIterator[RequestOutput], request_id: str
109
- ) -> Union[ErrorResponse, AsyncGenerator[str, None]]:
110
-
142
+ result_generator: AsyncIterator[RequestOutput],
143
+ request_id: str) -> AsyncGenerator[str, None]:
144
model_name = self.served_model_names[0]
145
created_time = int(time.time())
146
chunk_object_type = "chat.completion.chunk"
@@ -252,7 +285,7 @@ class OpenAIServingChat(OpenAIServing):
285
286
model_name = self.served_model_names[0]
287
created_time = int(time.time())
255
- final_res: RequestOutput = None
288
+ final_res: Optional[RequestOutput] = None
289
290
async for res in result_generator:
291
if await raw_request.is_disconnected():
@@ -317,7 +350,7 @@ class OpenAIServingChat(OpenAIServing):
350
351
return response
352
320
- def _load_chat_template(self, chat_template):
353
+ def _load_chat_template(self, chat_template: Optional[str]):
354
tokenizer = self.tokenizer
355
356
if chat_template is not None:
vllm/entrypoints/openai/serving_completion.py
+5
-4
@@ -11,7 +11,8 @@ from vllm.entrypoints.openai.protocol import (CompletionRequest,
11
CompletionResponseStreamChoice,
12
CompletionStreamResponse,
13
LogProbs, UsageInfo)
14
-from vllm.entrypoints.openai.serving_engine import LoRA, OpenAIServing
14
+from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
15
+ OpenAIServing)
16
from vllm.logger import init_logger
17
from vllm.model_executor.guided_decoding import (
18
get_guided_decoding_logits_processor)
@@ -54,7 +55,7 @@ class OpenAIServingCompletion(OpenAIServing):
55
def __init__(self,
56
engine: AsyncLLMEngine,
57
served_model_names: List[str],
57
- lora_modules: Optional[List[LoRA]] = None):
58
+ lora_modules: Optional[List[LoRAModulePath]] = None):
59
super().__init__(engine=engine,
60
served_model_names=served_model_names,
61
lora_modules=lora_modules)
@@ -84,7 +85,7 @@ class OpenAIServingCompletion(OpenAIServing):
85
created_time = int(time.time())
86
87
# Schedule the request and get the result generator.
87
- generators = []
88
+ generators: List[AsyncIterator[RequestOutput]] = []
89
try:
90
sampling_params = request.to_sampling_params()
91
lora_request = self._maybe_get_lora(request)
@@ -148,7 +149,7 @@ class OpenAIServingCompletion(OpenAIServing):
149
num_prompts=len(prompts))
150
151
# Non-streaming response
151
- final_res_batch: RequestOutput = [None] * len(prompts)
152
+ final_res_batch: List[Optional[RequestOutput]] = [None] * len(prompts)
153
try:
154
async for i, res in result_generator:
155
if await raw_request.is_disconnected():
vllm/entrypoints/openai/serving_engine.py
+10
-8
@@ -22,17 +22,15 @@ logger = init_logger(__name__)
22
23
24
@dataclass
25
-class LoRA:
25
+class LoRAModulePath:
26
name: str
27
local_path: str
28
29
30
class OpenAIServing:
31
32
- def __init__(self,
33
- engine: AsyncLLMEngine,
34
- served_model_names: List[str],
35
- lora_modules=Optional[List[LoRA]]):
32
+ def __init__(self, engine: AsyncLLMEngine, served_model_names: List[str],
33
+ lora_modules: Optional[List[LoRAModulePath]]):
34
self.engine = engine
35
self.served_model_names = served_model_names
36
if lora_modules is None:
@@ -158,7 +156,9 @@ class OpenAIServing:
156
})
157
return json_str
158
161
- async def _check_model(self, request) -> Optional[ErrorResponse]:
159
+ async def _check_model(
160
+ self, request: Union[CompletionRequest, ChatCompletionRequest]
161
+ ) -> Optional[ErrorResponse]:
162
if request.model in self.served_model_names:
163
return None
164
if request.model in [lora.lora_name for lora in self.lora_requests]:
@@ -168,14 +168,16 @@ class OpenAIServing:
168
err_type="NotFoundError",
169
status_code=HTTPStatus.NOT_FOUND)
170
171
- def _maybe_get_lora(self, request) -> Optional[LoRARequest]:
171
+ def _maybe_get_lora(
172
+ self, request: Union[CompletionRequest, ChatCompletionRequest]
173
+ ) -> Optional[LoRARequest]:
174
if request.model in self.served_model_names:
175
return None
176
for lora in self.lora_requests:
177
if request.model == lora.lora_name:
178
return lora
179
# if _check_model has been called earlier, this will be unreachable
178
- raise ValueError("The model `{request.model}` does not exist.")
180
+ raise ValueError(f"The model `{request.model}` does not exist.")
181
182
def _validate_prompt_and_tokenize(
183
self,