@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4c2b38ce9e

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 08:30 UTC

Enable Pydantic mypy checks and convert configs to Pydantic dataclasses (#17599)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed May 28, 2025 at 13:46 UTC 4c2b38ce9e90a0ac7c3e7ca400daf3a622cc7bca

11 files changed +115 -102

.pre-commit-config.yaml

+1 -1

@@ -58,7 +58,7 @@ repos:

58 entry: tools/mypy.sh 0 "local"

59 language: python

60 types: [python]

61 - additional_dependencies: &mypy_deps [mypy==1.11.1, types-cachetools, types-setuptools, types-PyYAML, types-requests]

61 + additional_dependencies: &mypy_deps [mypy==1.11.1, types-cachetools, types-setuptools, types-PyYAML, types-requests, pydantic]

62 stages: [pre-commit] # Don't run in CI

63 - id: mypy-3.9 # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup is less awkward

64 name: Run mypy for Python 3.9

pyproject.toml

@@ -110,6 +110,7 @@ ignore = [

110 ]

111

112 [tool.mypy]

113 +plugins = ['pydantic.mypy']

114 ignore_missing_imports = true

115 check_untyped_defs = true

116 follow_imports = "silent"

tests/lora/test_quant_model.py

+7 -7

@@ -24,16 +24,16 @@ if current_platform.is_rocm():

24 MODELS = [

25 ModelWithQuantization(

26 model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ",

27 - quantization="GPTQ"),

27 + quantization="gptq"),

28 ]

29 else:

30 MODELS = [

31 ModelWithQuantization(

32 model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ",

33 - quantization="AWQ"),

33 + quantization="awq"),

34 ModelWithQuantization(

35 model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ",

36 - quantization="GPTQ"),

36 + quantization="gptq"),

37 ]

@@ -100,7 +100,7 @@ def test_quant_model_lora(tinyllama_lora_files, model):

100 "#ff8050",

101 "#ff8080",

102 ]

103 - elif model.quantization == "AWQ":

103 + elif model.quantization == "awq":

104 expected_no_lora_output = [

105 "I'm sorry, I don't understand",

106 "I'm sorry, I don't understand",

@@ -109,7 +109,7 @@ def test_quant_model_lora(tinyllama_lora_files, model):

109 "#f07700: A v",

110 "#f00000: A v",

111 ]

112 - elif model.quantization == "GPTQ":

112 + elif model.quantization == "gptq":

113 expected_no_lora_output = [

114 "I'm sorry, I don't have",

115 "I'm sorry, I don't have",

@@ -122,7 +122,7 @@ def test_quant_model_lora(tinyllama_lora_files, model):

122 def expect_match(output, expected_output):

123 # HACK: GPTQ lora outputs are just incredibly unstable.

124 # Assert that the outputs changed.

125 - if (model.quantization == "GPTQ"

125 + if (model.quantization == "gptq"

126 and expected_output is expected_lora_output):

127 assert output != expected_no_lora_output

128 for i, o in enumerate(output):

@@ -172,7 +172,7 @@ def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available,

172 model):

173 if num_gpus_available < 2:

174 pytest.skip(f"Not enough GPUs for tensor parallelism {2}")

175 - if model.quantization == "GPTQ":

175 + if model.quantization == "gptq":

176 pytest.skip("GPTQ lora outputs are just incredibly unstable")

177 llm_tp1 = vllm.LLM(

178 model=model.model_path,

tests/tracing/test_tracing.py

+1 -1

@@ -173,7 +173,7 @@ def test_traces_with_detailed_steps(

173 llm = LLM(

174 model=model,

175 otlp_traces_endpoint=FAKE_TRACE_SERVER_ADDRESS,

176 - collect_detailed_traces="all",

176 + collect_detailed_traces=["all"],

177 )

178 prompts = ["This is a short prompt"]

179 outputs = llm.generate(prompts, sampling_params=sampling_params)

vllm/config.py

+60 -48

@@ -11,8 +11,8 @@ import uuid

11 import warnings

12 from collections import Counter

13 from contextlib import contextmanager

14 -from dataclasses import (MISSING, Field, asdict, dataclass, field, fields,

15 - is_dataclass, replace)

14 +from dataclasses import (MISSING, Field, asdict, field, fields, is_dataclass,

15 + replace)

16 from functools import cached_property

17 from importlib.util import find_spec

18 from pathlib import Path

@@ -21,9 +21,12 @@ from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Literal, Optional,

22 import regex as re

23 import torch

24 +from pydantic import (ConfigDict, SkipValidation, TypeAdapter, field_validator,

25 + model_validator)

26 +from pydantic.dataclasses import dataclass

27 from torch.distributed import ProcessGroup, ReduceOp

28 from transformers import PretrainedConfig

26 -from typing_extensions import deprecated

29 +from typing_extensions import deprecated, runtime_checkable

31 import vllm.envs as envs

32 from vllm import version

@@ -57,10 +60,15 @@ if TYPE_CHECKING:

60 from vllm.model_executor.layers.quantization.base_config import (

61 QuantizationConfig)

62 from vllm.model_executor.model_loader import BaseModelLoader

63 + from vllm.model_executor.model_loader.tensorizer import TensorizerConfig

65 ConfigType = type[DataclassInstance]

66 else:

67 + PlacementGroup = Any

68 + ExecutorBase = Any

69 QuantizationConfig = Any

70 + BaseModelLoader = Any

71 + TensorizerConfig = Any

72 ConfigType = type

74 logger = init_logger(__name__)

@@ -92,6 +100,7 @@ HfOverrides = Union[dict[str, Any], Callable[[PretrainedConfig],

100 PretrainedConfig]]

101

102

103 +@runtime_checkable

104 class SupportsHash(Protocol):

105

106 def compute_hash(self) -> str:

@@ -223,7 +232,7 @@ ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]

232

233

234 @config

226 -@dataclass

235 +@dataclass(config=ConfigDict(arbitrary_types_allowed=True))

236 class ModelConfig:

237 """Configuration for the model."""

238

@@ -236,7 +245,7 @@ class ModelConfig:

245 task, even if the same model can be used for multiple tasks. When the model

246 only supports one task, "auto" can be used to select it; otherwise, you

247 must specify explicitly which task to use."""

239 - tokenizer: str = None # type: ignore

248 + tokenizer: SkipValidation[str] = None # type: ignore

249 """Name or path of the Hugging Face tokenizer to use. If unspecified, model

250 name or path will be used."""

251 tokenizer_mode: TokenizerMode = "auto"

@@ -284,7 +293,7 @@ class ModelConfig:

293 """The specific revision to use for the tokenizer on the Hugging Face Hub.

294 It can be a branch name, a tag name, or a commit id. If unspecified, will

295 use the default version."""

287 - max_model_len: int = None # type: ignore

296 + max_model_len: SkipValidation[int] = None # type: ignore

297 """Model context length (prompt and output). If unspecified, will be

298 automatically derived from the model config.

299

@@ -602,6 +611,22 @@ class ModelConfig:

611 self._verify_cuda_graph()

612 self._verify_bnb_config()

613

614 + @field_validator("quantization", mode="before")

615 + @classmethod

616 + def validate_quantization_before(cls, value: Any) -> Any:

617 + if isinstance(value, str):

618 + return value.lower()

619 + return value

620 +

621 + @model_validator(mode="after")

622 + def validate_model_config_after(self: "ModelConfig") -> "ModelConfig":

623 + if not isinstance(self.tokenizer, str):

624 + raise ValueError("tokenizer must be a string after __post_init__.")

625 + if not isinstance(self.max_model_len, int):

626 + raise ValueError(

627 + "max_model_len must be an integer after __post_init__.")

628 + return self

629 +

630 @property

631 def registry(self):

632 return ModelRegistry

@@ -823,8 +848,7 @@ class ModelConfig:

848 "quark", "modelopt_fp4", "bitblas", "gptq_bitblas"

849 ]

850 if self.quantization is not None:

826 - self.quantization = cast(QuantizationMethods,

827 - self.quantization.lower())

851 + self.quantization = cast(QuantizationMethods, self.quantization)

852

853 # Parse quantization method from the HF model config, if available.

854 quant_cfg = self._parse_quant_hf_config()

@@ -1397,7 +1421,7 @@ PrefixCachingHashAlgo = Literal["builtin", "sha256"]

1421 class CacheConfig:

1422 """Configuration for the KV cache."""

1423

1400 - block_size: BlockSize = None # type: ignore

1424 + block_size: SkipValidation[BlockSize] = None # type: ignore

1425 """Size of a contiguous cache block in number of tokens. This is ignored on

1426 neuron devices and set to `--max-model-len`. On CUDA devices, only block

1427 sizes up to 32 are supported. On HPU devices, block size defaults to 128.

@@ -1619,7 +1643,8 @@ class LoadConfig:

1643 download_dir: Optional[str] = None

1644 """Directory to download and load the weights, default to the default

1645 cache directory of Hugging Face."""

1622 - model_loader_extra_config: dict = field(default_factory=dict)

1646 + model_loader_extra_config: Union[dict, TensorizerConfig] = field(

1647 + default_factory=dict)

1648 """Extra config for model loader. This will be passed to the model loader

1649 corresponding to the chosen load_format."""

1650 ignore_patterns: Optional[Union[list[str], str]] = None

@@ -1929,19 +1954,19 @@ class SchedulerConfig:

1954 runner_type: RunnerType = "generate"

1955 """The runner type to launch for the model."""

1956

1932 - max_num_batched_tokens: int = None # type: ignore

1957 + max_num_batched_tokens: SkipValidation[int] = None # type: ignore

1958 """Maximum number of tokens to be processed in a single iteration.

1959

1960 This config has no static default. If left unspecified by the user, it will

1961 be set in `EngineArgs.create_engine_config` based on the usage context."""

1962

1938 - max_num_seqs: int = None # type: ignore

1963 + max_num_seqs: SkipValidation[int] = None # type: ignore

1964 """Maximum number of sequences to be processed in a single iteration.

1965

1966 This config has no static default. If left unspecified by the user, it will

1967 be set in `EngineArgs.create_engine_config` based on the usage context."""

1968

1944 - max_model_len: int = None # type: ignore

1969 + max_model_len: SkipValidation[int] = None # type: ignore

1970 """Maximum length of a sequence (including prompt and generated text). This

1971 is primarily set in `ModelConfig` and that value should be manually

1972 duplicated here."""

@@ -1980,7 +2005,7 @@ class SchedulerConfig:

2005 """Apply a delay (of delay factor multiplied by previous

2006 prompt latency) before scheduling next prompt."""

2007

1983 - enable_chunked_prefill: bool = None # type: ignore

2008 + enable_chunked_prefill: SkipValidation[bool] = None # type: ignore

2009 """If True, prefill requests can be chunked based

2010 on the remaining max_num_batched_tokens."""

2011

@@ -2202,7 +2227,7 @@ Device = Literal["auto", "cuda", "neuron", "cpu", "tpu", "xpu", "hpu"]

2227

2228

2229 @config

2205 -@dataclass

2230 +@dataclass(config=ConfigDict(arbitrary_types_allowed=True))

2231 class DeviceConfig:

2232 """Configuration for the device to use for vLLM execution."""

2233

@@ -2260,8 +2285,8 @@ class DeviceConfig:

2285 self.device = torch.device(self.device_type)

2286

2287

2263 -SpeculativeMethod = Literal["ngram", "eagle", "medusa", "mlp_speculator",

2264 - "draft_model", "deepseek_mtp"]

2288 +SpeculativeMethod = Literal["ngram", "eagle", "eagle3", "medusa",

2289 + "mlp_speculator", "draft_model", "deepseek_mtp"]

2290 SpeculativeAcceptanceMethod = Literal["rejection_sampler",

2291 "typical_acceptance_sampler"]

2292

@@ -2272,8 +2297,7 @@ class SpeculativeConfig:

2297 """Configuration for speculative decoding."""

2298

2299 # General speculative decoding control

2275 - num_speculative_tokens: int = field(default=None,

2276 - init=True) # type: ignore

2300 + num_speculative_tokens: SkipValidation[int] = None # type: ignore

2301 """The number of speculative tokens, if provided. It will default to the

2302 number in the draft model config if present, otherwise, it is required."""

2303 model: Optional[str] = None

@@ -2349,26 +2373,23 @@ class SpeculativeConfig:

2373 """Specifies the tree structure for speculative token generation.

2374 """

2375 # required configuration params passed from engine

2352 - target_model_config: ModelConfig = field(default=None,

2353 - init=True) # type: ignore

2376 + target_model_config: SkipValidation[ModelConfig] = None # type: ignore

2377 """The configuration of the target model."""

2355 - target_parallel_config: ParallelConfig = field(default=None,

2356 - init=True) # type: ignore

2378 + target_parallel_config: SkipValidation[

2379 + ParallelConfig] = None # type: ignore

2380 """The parallel configuration for the target model."""

2358 - enable_chunked_prefill: bool = field(default=None,

2359 - init=True) # type: ignore

2381 + enable_chunked_prefill: SkipValidation[bool] = None # type: ignore

2382 """Whether vLLM is configured to use chunked prefill or not. Used for

2383 raising an error since it's not yet compatible with speculative decode."""

2362 - disable_log_stats: bool = field(default=None, init=True) # type: ignore

2384 + disable_log_stats: SkipValidation[bool] = None # type: ignore

2385 """Whether to disable the periodic printing of stage times in speculative

2386 decoding."""

2387

2388 # params generated in the post-init stage

2367 - draft_model_config: ModelConfig = field(default=None,

2368 - init=True) # type: ignore

2389 + draft_model_config: SkipValidation[ModelConfig] = None # type: ignore

2390 """The configuration of the draft model initialized internal."""

2370 - draft_parallel_config: ParallelConfig = field(default=None,

2371 - init=True) # type: ignore

2391 + draft_parallel_config: SkipValidation[

2392 + ParallelConfig] = None # type: ignore

2393 """The parallel configuration for the draft model initialized internal."""

2394

2395 def compute_hash(self) -> str:

@@ -2766,7 +2787,7 @@ LoRADType = Literal["auto", "float16", "bfloat16"]

2787

2788

2789 @config

2769 -@dataclass

2790 +@dataclass(config=ConfigDict(arbitrary_types_allowed=True))

2791 class LoRAConfig:

2792 """Configuration for LoRA."""

2793

@@ -2863,7 +2884,7 @@ class LoRAConfig:

2884

2885

2886 @config

2866 -@dataclass

2887 +@dataclass(config=ConfigDict(arbitrary_types_allowed=True))

2888 class PromptAdapterConfig:

2889 """Configuration for PromptAdapters."""

2890

@@ -3892,17 +3913,11 @@ class CompilationConfig:

3913 "pass_config",

3914 "traced_files",

3915 }

3895 - include = dict()

3896 - for k, v in asdict(self).items():

3897 - if k in exclude:

3898 - continue

3899 - f = get_field(CompilationConfig, k)

3900 - if (d := f.default) is not MISSING and d == v:

3901 - continue

3902 - if (df := f.default_factory) is not MISSING and df() == v:

3903 - continue

3904 - include[k] = v

3905 - return json.dumps(include)

3916 + # The cast to string is necessary because Pydantic is mocked in docs

3917 + # builds and sphinx-argparse doesn't know the return type of decode()

3918 + return str(

3919 + TypeAdapter(CompilationConfig).dump_json(

3920 + self, exclude=exclude, exclude_unset=True).decode())

3921

3922 __str__ = __repr__

3923

@@ -3911,7 +3926,7 @@ class CompilationConfig:

3926 """Parse the CLI value for the compilation config."""

3927 if cli_value in ["0", "1", "2", "3"]:

3928 return cls(level=int(cli_value))

3914 - return cls(**json.loads(cli_value))

3929 + return TypeAdapter(CompilationConfig).validate_json(cli_value)

3930

3931 def __post_init__(self) -> None:

3932 count_none = self.custom_ops.count("none")

@@ -4037,7 +4052,7 @@ class CompilationConfig:

4052

4053

4054 @config

4040 -@dataclass

4055 +@dataclass(config=ConfigDict(arbitrary_types_allowed=True))

4056 class VllmConfig:

4057 """Dataclass which contains all vllm-related configuration. This

4058 simplifies passing around the distinct configurations in the codebase.

@@ -4294,9 +4309,6 @@ class VllmConfig:

4309 "To workaround this limitation, vLLM will set 'ieee' input "

4310 "precision for chunked prefill triton kernels.")

4311

4297 - if self.compilation_config is None:

4298 - self.compilation_config = CompilationConfig()

4299 -

4312 # async tp is built on top of sequence parallelism

4313 # and requires it to be enabled.

4314 if self.compilation_config.pass_config.enable_async_tp:

vllm/engine/arg_utils.py

+23 -20

@@ -14,6 +14,7 @@ from typing import (Annotated, Any, Callable, Dict, List, Literal, Optional,

15 import regex as re

16 import torch

17 +from pydantic import SkipValidation, TypeAdapter, ValidationError

18 from typing_extensions import TypeIs, deprecated

20 import vllm.envs as envs

@@ -38,7 +39,7 @@ from vllm.test_utils import MODEL_WEIGHTS_S3_BUCKET, MODELS_ON_S3

39 from vllm.transformers_utils.utils import check_gguf_file

40 from vllm.usage.usage_lib import UsageContext

41 from vllm.utils import (STR_DUAL_CHUNK_FLASH_ATTN_VAL, FlexibleArgumentParser,

41 - GiB_bytes, is_in_doc_build, is_in_ray_actor)

42 + GiB_bytes, is_in_ray_actor)

44 # yapf: enable

@@ -156,7 +157,8 @@ def get_kwargs(cls: ConfigType) -> dict[str, Any]:

157 # Get the set of possible types for the field

158 type_hints: set[TypeHint] = set()

159 if get_origin(field.type) in {Union, Annotated}:

159 - type_hints.update(get_args(field.type))

160 + predicate = lambda arg: not isinstance(arg, SkipValidation)

161 + type_hints.update(filter(predicate, get_args(field.type)))

162 else:

163 type_hints.add(field.type)

164

@@ -168,10 +170,7 @@ def get_kwargs(cls: ConfigType) -> dict[str, Any]:

170 if field.default is not MISSING:

171 default = field.default

172 elif field.default_factory is not MISSING:

171 - if is_dataclass(field.default_factory) and is_in_doc_build():

172 - default = {}

173 - else:

174 - default = field.default_factory()

173 + default = field.default_factory()

174

175 # Get the help text for the field

176 name = field.name

@@ -189,12 +188,16 @@ def get_kwargs(cls: ConfigType) -> dict[str, Any]:

188 - `--json-arg '{"key1": "value1", "key2": {"key3": "value2"}}'`\n

189 - `--json-arg.key1 value1 --json-arg.key2.key3 value2`\n\n"""

190 if dataclass_cls is not None:

192 - dataclass_init = lambda x, f=dataclass_cls: f(**json.loads(x))

193 - # Special case for configs with a from_cli method

194 - if hasattr(dataclass_cls, "from_cli"):

195 - from_cli = dataclass_cls.from_cli

196 - dataclass_init = lambda x, f=from_cli: f(x)

197 - kwargs[name]["type"] = dataclass_init

191 +

192 + def parse_dataclass(val: str, cls=dataclass_cls) -> Any:

193 + try:

194 + if hasattr(cls, "from_cli"):

195 + return cls.from_cli(val)

196 + return TypeAdapter(cls).validate_json(val)

197 + except ValidationError as e:

198 + raise argparse.ArgumentTypeError(repr(e)) from e

199 +

200 + kwargs[name]["type"] = parse_dataclass

201 kwargs[name]["help"] += json_tip

202 elif contains_type(type_hints, bool):

203 # Creates --no-<name> and --<name> flags

@@ -225,12 +228,11 @@ def get_kwargs(cls: ConfigType) -> dict[str, Any]:

228 kwargs[name]["type"] = human_readable_int

229 elif contains_type(type_hints, float):

230 kwargs[name]["type"] = float

228 - elif contains_type(type_hints,

229 - dict) and (contains_type(type_hints, str) or any(

230 - is_not_builtin(th) for th in type_hints)):

231 + elif (contains_type(type_hints, dict)

232 + and (contains_type(type_hints, str)

233 + or any(is_not_builtin(th) for th in type_hints))):

234 kwargs[name]["type"] = union_dict_and_str

235 elif contains_type(type_hints, dict):

233 - # Dict arguments will always be optional

236 kwargs[name]["type"] = parse_type(json.loads)

237 kwargs[name]["help"] += json_tip

238 elif (contains_type(type_hints, str)

@@ -317,8 +319,7 @@ class EngineArgs:

319 rope_scaling: dict[str, Any] = get_field(ModelConfig, "rope_scaling")

320 rope_theta: Optional[float] = ModelConfig.rope_theta

321 hf_token: Optional[Union[bool, str]] = ModelConfig.hf_token

320 - hf_overrides: Optional[HfOverrides] = \

321 - get_field(ModelConfig, "hf_overrides")

322 + hf_overrides: HfOverrides = get_field(ModelConfig, "hf_overrides")

323 tokenizer_revision: Optional[str] = ModelConfig.tokenizer_revision

324 quantization: Optional[QuantizationMethods] = ModelConfig.quantization

325 enforce_eager: bool = ModelConfig.enforce_eager

@@ -398,7 +399,8 @@ class EngineArgs:

399 get_field(ModelConfig, "override_neuron_config")

400 override_pooler_config: Optional[Union[dict, PoolerConfig]] = \

401 ModelConfig.override_pooler_config

401 - compilation_config: Optional[CompilationConfig] = None

402 + compilation_config: CompilationConfig = \

403 + get_field(VllmConfig, "compilation_config")

404 worker_cls: str = ParallelConfig.worker_cls

405 worker_extension_cls: str = ParallelConfig.worker_extension_cls

406

@@ -413,7 +415,8 @@ class EngineArgs:

415

416 calculate_kv_scales: bool = CacheConfig.calculate_kv_scales

417

416 - additional_config: Optional[Dict[str, Any]] = None

418 + additional_config: dict[str, Any] = \

419 + get_field(VllmConfig, "additional_config")

420 enable_reasoning: Optional[bool] = None # DEPRECATED

421 reasoning_parser: str = DecodingConfig.reasoning_backend

422

vllm/entrypoints/llm.py

+4 -1

@@ -207,6 +207,9 @@ class LLM:

207 if isinstance(worker_cls, type):

208 kwargs["worker_cls"] = cloudpickle.dumps(worker_cls)

209

210 + if hf_overrides is None:

211 + hf_overrides = {}

212 +

213 if compilation_config is not None:

214 if isinstance(compilation_config, int):

215 compilation_config_instance = CompilationConfig(

@@ -218,7 +221,7 @@ class LLM:

221 else:

222 compilation_config_instance = compilation_config

223 else:

221 - compilation_config_instance = None

224 + compilation_config_instance = CompilationConfig()

225

226 engine_args = EngineArgs(

227 model=model,

vllm/entrypoints/openai/protocol.py

+8 -4

@@ -175,11 +175,15 @@ class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):

175 type: Literal["function"] = "function"

176

177

178 +# extra="forbid" is a workaround to have kwargs as a field,

179 +# see https://github.com/pydantic/pydantic/issues/3125

180 class LogitsProcessorConstructor(BaseModel):

181 qualname: str

182 args: Optional[list[Any]] = None

183 kwargs: Optional[dict[str, Any]] = None

184

185 + model_config = ConfigDict(extra="forbid")

186 +

187

188 LogitsProcessors = list[Union[str, LogitsProcessorConstructor]]

189

@@ -234,7 +238,7 @@ class ChatCompletionRequest(OpenAIBaseModel):

238 presence_penalty: Optional[float] = 0.0

239 response_format: Optional[AnyResponseFormat] = None

240 seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)

237 - stop: Optional[Union[str, list[str]]] = Field(default_factory=list)

241 + stop: Optional[Union[str, list[str]]] = []

242 stream: Optional[bool] = False

243 stream_options: Optional[StreamOptions] = None

244 temperature: Optional[float] = None

@@ -258,7 +262,7 @@ class ChatCompletionRequest(OpenAIBaseModel):

262 min_p: Optional[float] = None

263 repetition_penalty: Optional[float] = None

264 length_penalty: float = 1.0

261 - stop_token_ids: Optional[list[int]] = Field(default_factory=list)

265 + stop_token_ids: Optional[list[int]] = []

266 include_stop_str_in_output: bool = False

267 ignore_eos: bool = False

268 min_tokens: int = 0

@@ -756,7 +760,7 @@ class CompletionRequest(OpenAIBaseModel):

760 n: int = 1

761 presence_penalty: Optional[float] = 0.0

762 seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)

759 - stop: Optional[Union[str, list[str]]] = Field(default_factory=list)

763 + stop: Optional[Union[str, list[str]]] = []

764 stream: Optional[bool] = False

765 stream_options: Optional[StreamOptions] = None

766 suffix: Optional[str] = None

@@ -770,7 +774,7 @@ class CompletionRequest(OpenAIBaseModel):

774 min_p: Optional[float] = None

775 repetition_penalty: Optional[float] = None

776 length_penalty: float = 1.0

773 - stop_token_ids: Optional[list[int]] = Field(default_factory=list)

777 + stop_token_ids: Optional[list[int]] = []

778 include_stop_str_in_output: bool = False

779 ignore_eos: bool = False

780 min_tokens: int = 0

vllm/entrypoints/openai/serving_engine.py

+8 -8

@@ -134,11 +134,9 @@ class RequestProcessingMixin(BaseModel):

134 Mixin for request processing,

135 handling prompt preparation and engine input.

136 """

137 - request_prompts: Optional[Sequence[RequestPrompt]] = \

138 - Field(default_factory=list)

137 + request_prompts: Optional[Sequence[RequestPrompt]] = []

138 engine_prompts: Optional[Union[list[EngineTokensPrompt],

140 - list[EngineEmbedsPrompt]]] = Field(

141 - default_factory=list)

139 + list[EngineEmbedsPrompt]]] = []

140

141 model_config = ConfigDict(arbitrary_types_allowed=True)

142

@@ -528,12 +526,14 @@ class OpenAIServing:

526 if isinstance(request,

527 (EmbeddingChatRequest, EmbeddingCompletionRequest,

528 ScoreRequest, RerankRequest, ClassificationRequest)):

531 - operation = {

532 - ScoreRequest: "score",

533 - ClassificationRequest: "classification"

534 - }.get(type(request), "embedding generation")

529

530 if token_num > self.max_model_len:

531 + operations: dict[type[AnyRequest], str] = {

532 + ScoreRequest: "score",

533 + ClassificationRequest: "classification"

534 + }

535 + operation = operations.get(type(request),

536 + "embedding generation")

537 raise ValueError(

538 f"This model's maximum context length is "

539 f"{self.max_model_len} tokens. However, you requested "

vllm/model_executor/guided_decoding/guided_fields.py

+2 -4

@@ -3,12 +3,10 @@

3 from dataclasses import dataclass

4 from typing import Optional, TypedDict, Union

6 -from pydantic import BaseModel

7 -

7 # These classes are deprecated, see SamplingParams

8 class LLMGuidedOptions(TypedDict, total=False):

11 - guided_json: Union[dict, BaseModel, str]

9 + guided_json: Union[dict, str]

10 guided_regex: str

11 guided_choice: list[str]

12 guided_grammar: str

@@ -20,7 +18,7 @@ class LLMGuidedOptions(TypedDict, total=False):

18 @dataclass

19 class GuidedDecodingRequest:

20 """One of the fields will be used to retrieve the logit processor."""

23 - guided_json: Optional[Union[dict, BaseModel, str]] = None

21 + guided_json: Optional[Union[dict, str]] = None

22 guided_regex: Optional[str] = None

23 guided_choice: Optional[list[str]] = None

24 guided_grammar: Optional[str] = None

vllm/utils.py

-8

@@ -1878,14 +1878,6 @@ def get_cuda_view_from_cpu_tensor(cpu_tensor: torch.Tensor) -> torch.Tensor:

1878 return torch.ops._C.get_cuda_view_from_cpu_tensor(cpu_tensor)

1879

1880

1881 -def is_in_doc_build() -> bool:

1882 - try:

1883 - from sphinx.ext.autodoc.mock import _MockModule

1884 - return isinstance(zmq, _MockModule)

1885 - except ModuleNotFoundError:

1886 - return False

1887 -

1888 -

1881 def import_from_path(module_name: str, file_path: Union[str, os.PathLike]):

1882 """

1883 Import a Python file according to its file path.