@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8947bc3c15
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Frontend][Bugfix] Disallow extra fields in OpenAI API (#4355)

Cyrus Leung committed Apr 27, 2024 at 13:08 UTC 8947bc3c156963dfc66e7ca1e4c436506ed6a512
8 files changed +113 -55
requirements-common.txt
+1
@@ -8,6 +8,7 @@ py-cpuinfo
8 transformers >= 4.40.0 # Required for StarCoder2 & Llava, Llama 3.
9 tokenizers >= 0.19.1 # Required for Llama 3.
10 fastapi
11 +openai
12 uvicorn[standard]
13 pydantic >= 2.0 # Required for OpenAI server.
14 prometheus_client >= 0.18.0
requirements-dev.txt
-1
@@ -21,7 +21,6 @@ pytest-rerunfailures
21 pytest-shard
22 httpx
23 einops # required for MPT
24 -openai
24 requests
25 ray
26 peft
tests/entrypoints/test_openai_server.py
+16
@@ -15,6 +15,7 @@ import ray
15 import requests
16 # downloading lora to test lora requests
17 from huggingface_hub import snapshot_download
18 +from openai import BadRequestError
19
20 from vllm.transformers_utils.tokenizer import get_tokenizer
21
@@ -770,6 +771,21 @@ async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
771 assert loaded == {"result": 2}, loaded
772
773
774 +async def test_extra_fields(server, client: openai.AsyncOpenAI):
775 + with pytest.raises(BadRequestError) as exc_info:
776 + await client.chat.completions.create(
777 + model=MODEL_NAME,
778 + messages=[{
779 + "role": "system",
780 + "content": "You are a helpful assistant.",
781 + "extra_field": "0",
782 + }], # type: ignore
783 + temperature=0,
784 + seed=0)
785 +
786 + assert "extra_forbidden" in exc_info.value.message
787 +
788 +
789 async def test_guided_grammar(server, client: openai.AsyncOpenAI):
790 simple_sql_grammar = """
791 start: select_statement
vllm/entrypoints/openai/cli_args.py
+2 -2
@@ -9,7 +9,7 @@ import json
9 import ssl
10
11 from vllm.engine.arg_utils import AsyncEngineArgs
12 -from vllm.entrypoints.openai.serving_engine import LoRA
12 +from vllm.entrypoints.openai.serving_engine import LoRAModulePath
13
14
15 class LoRAParserAction(argparse.Action):
@@ -18,7 +18,7 @@ class LoRAParserAction(argparse.Action):
18 lora_list = []
19 for item in values:
20 name, path = item.split('=')
21 - lora_list.append(LoRA(name, path))
21 + lora_list.append(LoRAModulePath(name, path))
22 setattr(namespace, self.dest, lora_list)
23
24
vllm/entrypoints/openai/protocol.py
+35 -29
@@ -4,14 +4,20 @@ import time
4 from typing import Dict, List, Literal, Optional, Union
5
6 import torch
7 -from pydantic import BaseModel, Field, model_validator
7 +from openai.types.chat import ChatCompletionMessageParam
8 +from pydantic import BaseModel, ConfigDict, Field, model_validator
9 from typing_extensions import Annotated
10
11 from vllm.sampling_params import SamplingParams
12 from vllm.utils import random_uuid
13
14
14 -class ErrorResponse(BaseModel):
15 +class OpenAIBaseModel(BaseModel):
16 + # OpenAI API does not allow extra fields
17 + model_config = ConfigDict(extra="forbid")
18 +
19 +
20 +class ErrorResponse(OpenAIBaseModel):
21 object: str = "error"
22 message: str
23 type: str
@@ -19,7 +25,7 @@ class ErrorResponse(BaseModel):
25 code: int
26
27
22 -class ModelPermission(BaseModel):
28 +class ModelPermission(OpenAIBaseModel):
29 id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
30 object: str = "model_permission"
31 created: int = Field(default_factory=lambda: int(time.time()))
@@ -34,7 +40,7 @@ class ModelPermission(BaseModel):
40 is_blocking: bool = False
41
42
37 -class ModelCard(BaseModel):
43 +class ModelCard(OpenAIBaseModel):
44 id: str
45 object: str = "model"
46 created: int = Field(default_factory=lambda: int(time.time()))
@@ -44,26 +50,26 @@ class ModelCard(BaseModel):
50 permission: List[ModelPermission] = Field(default_factory=list)
51
52
47 -class ModelList(BaseModel):
53 +class ModelList(OpenAIBaseModel):
54 object: str = "list"
55 data: List[ModelCard] = Field(default_factory=list)
56
57
52 -class UsageInfo(BaseModel):
58 +class UsageInfo(OpenAIBaseModel):
59 prompt_tokens: int = 0
60 total_tokens: int = 0
61 completion_tokens: Optional[int] = 0
62
63
58 -class ResponseFormat(BaseModel):
64 +class ResponseFormat(OpenAIBaseModel):
65 # type must be "json_object" or "text"
66 type: Literal["text", "json_object"]
67
68
63 -class ChatCompletionRequest(BaseModel):
69 +class ChatCompletionRequest(OpenAIBaseModel):
70 # Ordered by official OpenAI API documentation
71 # https://platform.openai.com/docs/api-reference/chat/create
66 - messages: List[Dict[str, str]]
72 + messages: List[ChatCompletionMessageParam]
73 model: str
74 frequency_penalty: Optional[float] = 0.0
75 logit_bias: Optional[Dict[str, float]] = None
@@ -204,7 +210,7 @@ class ChatCompletionRequest(BaseModel):
210 return data
211
212
207 -class CompletionRequest(BaseModel):
213 +class CompletionRequest(OpenAIBaseModel):
214 # Ordered by official OpenAI API documentation
215 # https://platform.openai.com/docs/api-reference/completions/create
216 model: str
@@ -343,19 +349,19 @@ class CompletionRequest(BaseModel):
349 return data
350
351
346 -class LogProbs(BaseModel):
352 +class LogProbs(OpenAIBaseModel):
353 text_offset: List[int] = Field(default_factory=list)
354 token_logprobs: List[Optional[float]] = Field(default_factory=list)
355 tokens: List[str] = Field(default_factory=list)
356 top_logprobs: Optional[List[Optional[Dict[str, float]]]] = None
357
358
353 -class CompletionResponseChoice(BaseModel):
359 +class CompletionResponseChoice(OpenAIBaseModel):
360 index: int
361 text: str
362 logprobs: Optional[LogProbs] = None
357 - finish_reason: Optional[Literal["stop", "length"]] = None
358 - stop_reason: Union[None, int, str] = Field(
363 + finish_reason: Optional[str] = None
364 + stop_reason: Optional[Union[int, str]] = Field(
365 default=None,
366 description=(
367 "The stop string or token id that caused the completion "
@@ -364,7 +370,7 @@ class CompletionResponseChoice(BaseModel):
370 )
371
372
367 -class CompletionResponse(BaseModel):
373 +class CompletionResponse(OpenAIBaseModel):
374 id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
375 object: str = "text_completion"
376 created: int = Field(default_factory=lambda: int(time.time()))
@@ -373,12 +379,12 @@ class CompletionResponse(BaseModel):
379 usage: UsageInfo
380
381
376 -class CompletionResponseStreamChoice(BaseModel):
382 +class CompletionResponseStreamChoice(OpenAIBaseModel):
383 index: int
384 text: str
385 logprobs: Optional[LogProbs] = None
380 - finish_reason: Optional[Literal["stop", "length"]] = None
381 - stop_reason: Union[None, int, str] = Field(
386 + finish_reason: Optional[str] = None
387 + stop_reason: Optional[Union[int, str]] = Field(
388 default=None,
389 description=(
390 "The stop string or token id that caused the completion "
@@ -387,7 +393,7 @@ class CompletionResponseStreamChoice(BaseModel):
393 )
394
395
390 -class CompletionStreamResponse(BaseModel):
396 +class CompletionStreamResponse(OpenAIBaseModel):
397 id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
398 object: str = "text_completion"
399 created: int = Field(default_factory=lambda: int(time.time()))
@@ -396,20 +402,20 @@ class CompletionStreamResponse(BaseModel):
402 usage: Optional[UsageInfo] = Field(default=None)
403
404
399 -class ChatMessage(BaseModel):
405 +class ChatMessage(OpenAIBaseModel):
406 role: str
407 content: str
408
409
404 -class ChatCompletionResponseChoice(BaseModel):
410 +class ChatCompletionResponseChoice(OpenAIBaseModel):
411 index: int
412 message: ChatMessage
413 logprobs: Optional[LogProbs] = None
408 - finish_reason: Optional[Literal["stop", "length"]] = None
409 - stop_reason: Union[None, int, str] = None
414 + finish_reason: Optional[str] = None
415 + stop_reason: Optional[Union[int, str]] = None
416
417
412 -class ChatCompletionResponse(BaseModel):
418 +class ChatCompletionResponse(OpenAIBaseModel):
419 id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
420 object: str = "chat.completion"
421 created: int = Field(default_factory=lambda: int(time.time()))
@@ -418,20 +424,20 @@ class ChatCompletionResponse(BaseModel):
424 usage: UsageInfo
425
426
421 -class DeltaMessage(BaseModel):
427 +class DeltaMessage(OpenAIBaseModel):
428 role: Optional[str] = None
429 content: Optional[str] = None
430
431
426 -class ChatCompletionResponseStreamChoice(BaseModel):
432 +class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
433 index: int
434 delta: DeltaMessage
435 logprobs: Optional[LogProbs] = None
430 - finish_reason: Optional[Literal["stop", "length"]] = None
431 - stop_reason: Union[None, int, str] = None
436 + finish_reason: Optional[str] = None
437 + stop_reason: Optional[Union[int, str]] = None
438
439
434 -class ChatCompletionStreamResponse(BaseModel):
440 +class ChatCompletionStreamResponse(OpenAIBaseModel):
441 id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
442 object: str = "chat.completion.chunk"
443 created: int = Field(default_factory=lambda: int(time.time()))
vllm/entrypoints/openai/serving_chat.py
+44 -11
@@ -1,8 +1,11 @@
1 import codecs
2 import time
3 -from typing import AsyncGenerator, AsyncIterator, List, Optional, Union
3 +from typing import (AsyncGenerator, AsyncIterator, Awaitable, Iterable, List,
4 + Optional, Tuple, TypedDict, Union, final)
5
6 from fastapi import Request
7 +from openai.types.chat import (ChatCompletionContentPartParam,
8 + ChatCompletionRole)
9
10 from vllm.engine.async_llm_engine import AsyncLLMEngine
11 from vllm.entrypoints.openai.protocol import (
@@ -10,7 +13,8 @@ from vllm.entrypoints.openai.protocol import (
13 ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
14 ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
15 UsageInfo)
13 -from vllm.entrypoints.openai.serving_engine import LoRA, OpenAIServing
16 +from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
17 + OpenAIServing)
18 from vllm.logger import init_logger
19 from vllm.model_executor.guided_decoding import (
20 get_guided_decoding_logits_processor)
@@ -20,20 +24,41 @@ from vllm.utils import random_uuid
24 logger = init_logger(__name__)
25
26
27 +@final # So that it should be compatible with Dict[str, str]
28 +class ConversationMessage(TypedDict):
29 + role: str
30 + content: str
31 +
32 +
33 class OpenAIServingChat(OpenAIServing):
34
35 def __init__(self,
36 engine: AsyncLLMEngine,
37 served_model_names: List[str],
38 response_role: str,
29 - lora_modules: Optional[List[LoRA]] = None,
30 - chat_template=None):
39 + lora_modules: Optional[List[LoRAModulePath]] = None,
40 + chat_template: Optional[str] = None):
41 super().__init__(engine=engine,
42 served_model_names=served_model_names,
43 lora_modules=lora_modules)
44 self.response_role = response_role
45 self._load_chat_template(chat_template)
46
47 + def _parse_chat_message_content(
48 + self,
49 + role: ChatCompletionRole,
50 + content: Optional[Union[str,
51 + Iterable[ChatCompletionContentPartParam]]],
52 + ) -> Tuple[List[ConversationMessage], List[Awaitable[object]]]:
53 + if content is None:
54 + return [], []
55 + if isinstance(content, str):
56 + return [ConversationMessage(role=role, content=content)], []
57 +
58 + # To be implemented: https://github.com/vllm-project/vllm/pull/3467
59 + # To be implemented: https://github.com/vllm-project/vllm/pull/4200
60 + raise NotImplementedError("Complex input not supported yet")
61 +
62 async def create_chat_completion(
63 self, request: ChatCompletionRequest, raw_request: Request
64 ) -> Union[ErrorResponse, AsyncGenerator[str, None],
@@ -52,10 +77,19 @@ class OpenAIServingChat(OpenAIServing):
77 return error_check_ret
78
79 try:
80 + conversation: List[ConversationMessage] = []
81 +
82 + for m in request.messages:
83 + messages, _ = self._parse_chat_message_content(
84 + m["role"], m["content"])
85 +
86 + conversation.extend(messages)
87 +
88 prompt = self.tokenizer.apply_chat_template(
56 - conversation=request.messages,
89 + conversation=conversation,
90 tokenize=False,
58 - add_generation_prompt=request.add_generation_prompt)
91 + add_generation_prompt=request.add_generation_prompt,
92 + )
93 except Exception as e:
94 logger.error("Error in applying chat template from request: %s", e)
95 return self.create_error_response(str(e))
@@ -105,9 +139,8 @@ class OpenAIServingChat(OpenAIServing):
139
140 async def chat_completion_stream_generator(
141 self, request: ChatCompletionRequest,
108 - result_generator: AsyncIterator[RequestOutput], request_id: str
109 - ) -> Union[ErrorResponse, AsyncGenerator[str, None]]:
110 -
142 + result_generator: AsyncIterator[RequestOutput],
143 + request_id: str) -> AsyncGenerator[str, None]:
144 model_name = self.served_model_names[0]
145 created_time = int(time.time())
146 chunk_object_type = "chat.completion.chunk"
@@ -252,7 +285,7 @@ class OpenAIServingChat(OpenAIServing):
285
286 model_name = self.served_model_names[0]
287 created_time = int(time.time())
255 - final_res: RequestOutput = None
288 + final_res: Optional[RequestOutput] = None
289
290 async for res in result_generator:
291 if await raw_request.is_disconnected():
@@ -317,7 +350,7 @@ class OpenAIServingChat(OpenAIServing):
350
351 return response
352
320 - def _load_chat_template(self, chat_template):
353 + def _load_chat_template(self, chat_template: Optional[str]):
354 tokenizer = self.tokenizer
355
356 if chat_template is not None:
vllm/entrypoints/openai/serving_completion.py
+5 -4
@@ -11,7 +11,8 @@ from vllm.entrypoints.openai.protocol import (CompletionRequest,
11 CompletionResponseStreamChoice,
12 CompletionStreamResponse,
13 LogProbs, UsageInfo)
14 -from vllm.entrypoints.openai.serving_engine import LoRA, OpenAIServing
14 +from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
15 + OpenAIServing)
16 from vllm.logger import init_logger
17 from vllm.model_executor.guided_decoding import (
18 get_guided_decoding_logits_processor)
@@ -54,7 +55,7 @@ class OpenAIServingCompletion(OpenAIServing):
55 def __init__(self,
56 engine: AsyncLLMEngine,
57 served_model_names: List[str],
57 - lora_modules: Optional[List[LoRA]] = None):
58 + lora_modules: Optional[List[LoRAModulePath]] = None):
59 super().__init__(engine=engine,
60 served_model_names=served_model_names,
61 lora_modules=lora_modules)
@@ -84,7 +85,7 @@ class OpenAIServingCompletion(OpenAIServing):
85 created_time = int(time.time())
86
87 # Schedule the request and get the result generator.
87 - generators = []
88 + generators: List[AsyncIterator[RequestOutput]] = []
89 try:
90 sampling_params = request.to_sampling_params()
91 lora_request = self._maybe_get_lora(request)
@@ -148,7 +149,7 @@ class OpenAIServingCompletion(OpenAIServing):
149 num_prompts=len(prompts))
150
151 # Non-streaming response
151 - final_res_batch: RequestOutput = [None] * len(prompts)
152 + final_res_batch: List[Optional[RequestOutput]] = [None] * len(prompts)
153 try:
154 async for i, res in result_generator:
155 if await raw_request.is_disconnected():
vllm/entrypoints/openai/serving_engine.py
+10 -8
@@ -22,17 +22,15 @@ logger = init_logger(__name__)
22
23
24 @dataclass
25 -class LoRA:
25 +class LoRAModulePath:
26 name: str
27 local_path: str
28
29
30 class OpenAIServing:
31
32 - def __init__(self,
33 - engine: AsyncLLMEngine,
34 - served_model_names: List[str],
35 - lora_modules=Optional[List[LoRA]]):
32 + def __init__(self, engine: AsyncLLMEngine, served_model_names: List[str],
33 + lora_modules: Optional[List[LoRAModulePath]]):
34 self.engine = engine
35 self.served_model_names = served_model_names
36 if lora_modules is None:
@@ -158,7 +156,9 @@ class OpenAIServing:
156 })
157 return json_str
158
161 - async def _check_model(self, request) -> Optional[ErrorResponse]:
159 + async def _check_model(
160 + self, request: Union[CompletionRequest, ChatCompletionRequest]
161 + ) -> Optional[ErrorResponse]:
162 if request.model in self.served_model_names:
163 return None
164 if request.model in [lora.lora_name for lora in self.lora_requests]:
@@ -168,14 +168,16 @@ class OpenAIServing:
168 err_type="NotFoundError",
169 status_code=HTTPStatus.NOT_FOUND)
170
171 - def _maybe_get_lora(self, request) -> Optional[LoRARequest]:
171 + def _maybe_get_lora(
172 + self, request: Union[CompletionRequest, ChatCompletionRequest]
173 + ) -> Optional[LoRARequest]:
174 if request.model in self.served_model_names:
175 return None
176 for lora in self.lora_requests:
177 if request.model == lora.lora_name:
178 return lora
179 # if _check_model has been called earlier, this will be unreachable
178 - raise ValueError("The model `{request.model}` does not exist.")
180 + raise ValueError(f"The model `{request.model}` does not exist.")
181
182 def _validate_prompt_and_tokenize(
183 self,