A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix]: allow extra fields in requests to openai compatible server (#10463)
Signed-off-by: Guillaume Calmettes <gcalmettes@scaleway.com>
Guillaume Calmettes committed
Nov 20, 2024 at 22:42 UTC
c68f7ede6a4aef0cd31f531b5d7ec22ab224de95
2 files changed
+29
-15
tests/entrypoints/openai/test_chat.py
+13
-13
@@ -899,19 +899,19 @@ async def test_response_format_json_schema(client: openai.AsyncOpenAI):
899
900
901
@pytest.mark.asyncio
902
-async def test_extra_fields(client: openai.AsyncOpenAI):
903
- with pytest.raises(BadRequestError) as exc_info:
904
- await client.chat.completions.create(
905
- model=MODEL_NAME,
906
- messages=[{
907
- "role": "system",
908
- "content": "You are a helpful assistant.",
909
- "extra_field": "0",
910
- }], # type: ignore
911
- temperature=0,
912
- seed=0)
913
-
914
- assert "extra_forbidden" in exc_info.value.message
902
+async def test_extra_fields_allowed(client: openai.AsyncOpenAI):
903
+ resp = await client.chat.completions.create(
904
+ model=MODEL_NAME,
905
+ messages=[{
906
+ "role": "user",
907
+ "content": "what is 1+1?",
908
+ "extra_field": "0",
909
+ }], # type: ignore
910
+ temperature=0,
911
+ seed=0)
912
+
913
+ content = resp.choices[0].message.content
914
+ assert content is not None
915
916
917
@pytest.mark.asyncio
vllm/entrypoints/openai/protocol.py
+16
-2
@@ -9,12 +9,15 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator
9
from typing_extensions import Annotated
10
11
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
12
+from vllm.logger import init_logger
13
from vllm.pooling_params import PoolingParams
14
from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
15
RequestOutputKind, SamplingParams)
16
from vllm.sequence import Logprob
17
from vllm.utils import random_uuid
18
19
+logger = init_logger(__name__)
20
+
21
# torch is mocked during docs generation,
22
# so we have to provide the values as literals
23
_MOCK_LONG_INFO = Namespace(min=-9223372036854775808, max=9223372036854775807)
@@ -35,8 +38,19 @@ assert _LONG_INFO.max == _MOCK_LONG_INFO.max
38
39
40
class OpenAIBaseModel(BaseModel):
38
- # OpenAI API does not allow extra fields
39
- model_config = ConfigDict(extra="forbid")
41
+ # OpenAI API does allow extra fields
42
+ model_config = ConfigDict(extra="allow")
43
+
44
+ @model_validator(mode="before")
45
+ @classmethod
46
+ def __log_extra_fields__(cls, data):
47
+ if isinstance(data, dict):
48
+ extra_fields = data.keys() - cls.model_fields.keys()
49
+ if extra_fields:
50
+ logger.warning(
51
+ "The following fields were present in the request "
52
+ "but ignored: %s", extra_fields)
53
+ return data
54
55
56
class ErrorResponse(OpenAIBaseModel):