@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c68f7ede6a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix]: allow extra fields in requests to openai compatible server (#10463)

Signed-off-by: Guillaume Calmettes <gcalmettes@scaleway.com>

Guillaume Calmettes committed Nov 20, 2024 at 22:42 UTC c68f7ede6a4aef0cd31f531b5d7ec22ab224de95
2 files changed +29 -15
tests/entrypoints/openai/test_chat.py
+13 -13
@@ -899,19 +899,19 @@ async def test_response_format_json_schema(client: openai.AsyncOpenAI):
899
900
901 @pytest.mark.asyncio
902 -async def test_extra_fields(client: openai.AsyncOpenAI):
903 - with pytest.raises(BadRequestError) as exc_info:
904 - await client.chat.completions.create(
905 - model=MODEL_NAME,
906 - messages=[{
907 - "role": "system",
908 - "content": "You are a helpful assistant.",
909 - "extra_field": "0",
910 - }], # type: ignore
911 - temperature=0,
912 - seed=0)
913 -
914 - assert "extra_forbidden" in exc_info.value.message
902 +async def test_extra_fields_allowed(client: openai.AsyncOpenAI):
903 + resp = await client.chat.completions.create(
904 + model=MODEL_NAME,
905 + messages=[{
906 + "role": "user",
907 + "content": "what is 1+1?",
908 + "extra_field": "0",
909 + }], # type: ignore
910 + temperature=0,
911 + seed=0)
912 +
913 + content = resp.choices[0].message.content
914 + assert content is not None
915
916
917 @pytest.mark.asyncio
vllm/entrypoints/openai/protocol.py
+16 -2
@@ -9,12 +9,15 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator
9 from typing_extensions import Annotated
10
11 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
12 +from vllm.logger import init_logger
13 from vllm.pooling_params import PoolingParams
14 from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
15 RequestOutputKind, SamplingParams)
16 from vllm.sequence import Logprob
17 from vllm.utils import random_uuid
18
19 +logger = init_logger(__name__)
20 +
21 # torch is mocked during docs generation,
22 # so we have to provide the values as literals
23 _MOCK_LONG_INFO = Namespace(min=-9223372036854775808, max=9223372036854775807)
@@ -35,8 +38,19 @@ assert _LONG_INFO.max == _MOCK_LONG_INFO.max
38
39
40 class OpenAIBaseModel(BaseModel):
38 - # OpenAI API does not allow extra fields
39 - model_config = ConfigDict(extra="forbid")
41 + # OpenAI API does allow extra fields
42 + model_config = ConfigDict(extra="allow")
43 +
44 + @model_validator(mode="before")
45 + @classmethod
46 + def __log_extra_fields__(cls, data):
47 + if isinstance(data, dict):
48 + extra_fields = data.keys() - cls.model_fields.keys()
49 + if extra_fields:
50 + logger.warning(
51 + "The following fields were present in the request "
52 + "but ignored: %s", extra_fields)
53 + return data
54
55
56 class ErrorResponse(OpenAIBaseModel):