@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ef001d98ef
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

Fix the pydantic logging validator (#12420)

Signed-off-by: Max de Bayser <mbayser@br.ibm.com>

Maximilien de Bayser committed Jan 29, 2025 at 04:53 UTC ef001d98ef36166ebacb48eab2e32eb738407b53
1 file changed +22 -7
vllm/entrypoints/openai/protocol.py
+22 -7
@@ -6,7 +6,8 @@ from argparse import Namespace
6 from typing import Any, ClassVar, Dict, List, Literal, Optional, Set, Union
7
8 import torch
9 -from pydantic import BaseModel, ConfigDict, Field, model_validator
9 +from pydantic import (BaseModel, ConfigDict, Field, TypeAdapter,
10 + ValidationInfo, field_validator, model_validator)
11 from typing_extensions import Annotated
12
13 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
@@ -45,14 +46,14 @@ class OpenAIBaseModel(BaseModel):
46 # Cache class field names
47 field_names: ClassVar[Optional[Set[str]]] = None
48
48 - @model_validator(mode="before")
49 + @model_validator(mode="wrap")
50 @classmethod
50 - def __log_extra_fields__(cls, data):
51 -
51 + def __log_extra_fields__(cls, data, handler):
52 + result = handler(data)
53 + if not isinstance(data, dict):
54 + return result
55 field_names = cls.field_names
56 if field_names is None:
54 - if not isinstance(data, dict):
55 - return data
57 # Get all class field names and their potential aliases
58 field_names = set()
59 for field_name, field in cls.model_fields.items():
@@ -67,7 +68,7 @@ class OpenAIBaseModel(BaseModel):
68 "The following fields were present in the request "
69 "but ignored: %s",
70 data.keys() - field_names)
70 - return data
71 + return result
72
73
74 class ErrorResponse(OpenAIBaseModel):
@@ -1287,6 +1288,20 @@ class BatchRequestInput(OpenAIBaseModel):
1288 # The parameters of the request.
1289 body: Union[ChatCompletionRequest, EmbeddingRequest, ScoreRequest]
1290
1291 + @field_validator('body', mode='plain')
1292 + @classmethod
1293 + def check_type_for_url(cls, value: Any, info: ValidationInfo):
1294 + # Use url to disambiguate models
1295 + url = info.data['url']
1296 + if url == "/v1/chat/completions":
1297 + return ChatCompletionRequest.model_validate(value)
1298 + if url == "/v1/embeddings":
1299 + return TypeAdapter(EmbeddingRequest).validate_python(value)
1300 + if url == "/v1/score":
1301 + return ScoreRequest.model_validate(value)
1302 + return TypeAdapter(Union[ChatCompletionRequest, EmbeddingRequest,
1303 + ScoreRequest]).validate_python(value)
1304 +
1305
1306 class BatchResponseData(OpenAIBaseModel):
1307 # HTTP status code of the response.