A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Feature] Use pydantic validation in lora.py and load.py configs (#26413)
Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
Simon Danielsson committed
Oct 9, 2025 at 11:38 UTC
e4791438eda469d4263e2ce2ab0236f61f3c6ebc
4 files changed
+48
-45
vllm/config/load.py
+17
-9
@@ -2,9 +2,9 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import hashlib
5
-from dataclasses import field
5
from typing import TYPE_CHECKING, Any, Optional, Union
6
7
+from pydantic import Field, field_validator
8
from pydantic.dataclasses import dataclass
9
10
from vllm.config.utils import config
@@ -64,7 +64,7 @@ class LoadConfig:
64
was quantized using torchao and saved using safetensors.
65
Needs torchao >= 0.14.0
66
"""
67
- model_loader_extra_config: Union[dict, TensorizerConfig] = field(
67
+ model_loader_extra_config: Union[dict, TensorizerConfig] = Field(
68
default_factory=dict
69
)
70
"""Extra config for model loader. This will be passed to the model loader
@@ -72,7 +72,9 @@ class LoadConfig:
72
device: Optional[str] = None
73
"""Device to which model weights will be loaded, default to
74
device_config.device"""
75
- ignore_patterns: Optional[Union[list[str], str]] = None
75
+ ignore_patterns: Union[list[str], str] = Field(
76
+ default_factory=lambda: ["original/**/*"]
77
+ )
78
"""The list of patterns to ignore when loading the model. Default to
79
"original/**/*" to avoid repeated loading of llama's checkpoints."""
80
use_tqdm_on_load: bool = True
@@ -107,12 +109,18 @@ class LoadConfig:
109
hash_str = hashlib.md5(str(factors).encode(), usedforsecurity=False).hexdigest()
110
return hash_str
111
110
- def __post_init__(self):
111
- self.load_format = self.load_format.lower()
112
- if self.ignore_patterns is not None and len(self.ignore_patterns) > 0:
112
+ @field_validator("load_format", mode="after")
113
+ def _lowercase_load_format(cls, load_format: str) -> str:
114
+ return load_format.lower()
115
+
116
+ @field_validator("ignore_patterns", mode="after")
117
+ def _validate_ignore_patterns(
118
+ cls, ignore_patterns: Union[list[str], str]
119
+ ) -> Union[list[str], str]:
120
+ if ignore_patterns != ["original/**/*"] and len(ignore_patterns) > 0:
121
logger.info(
122
"Ignoring the following patterns when downloading weights: %s",
115
- self.ignore_patterns,
123
+ ignore_patterns,
124
)
117
- else:
118
- self.ignore_patterns = ["original/**/*"]
125
+
126
+ return ignore_patterns
vllm/config/lora.py
+22
-35
@@ -5,8 +5,9 @@ import hashlib
5
from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union
6
7
import torch
8
-from pydantic import ConfigDict
8
+from pydantic import ConfigDict, Field, model_validator
9
from pydantic.dataclasses import dataclass
10
+from typing_extensions import Self
11
12
import vllm.envs as envs
13
from vllm.config.utils import config
@@ -23,6 +24,8 @@ else:
24
logger = init_logger(__name__)
25
26
LoRADType = Literal["auto", "float16", "bfloat16"]
27
+MaxLoRARanks = Literal[1, 8, 16, 32, 64, 128, 256, 320, 512]
28
+LoRAExtraVocabSize = Literal[256, 512]
29
30
31
@config
@@ -30,9 +33,9 @@ LoRADType = Literal["auto", "float16", "bfloat16"]
33
class LoRAConfig:
34
"""Configuration for LoRA."""
35
33
- max_lora_rank: int = 16
36
+ max_lora_rank: MaxLoRARanks = 16
37
"""Max LoRA rank."""
35
- max_loras: int = 1
38
+ max_loras: int = Field(default=1, ge=1)
39
"""Max number of LoRAs in a single batch."""
40
fully_sharded_loras: bool = False
41
"""By default, only half of the LoRA computation is sharded with tensor
@@ -44,7 +47,14 @@ class LoRAConfig:
47
`max_loras`."""
48
lora_dtype: Union[torch.dtype, LoRADType] = "auto"
49
"""Data type for LoRA. If auto, will default to base model dtype."""
47
- lora_extra_vocab_size: int = 256
50
+ lora_extra_vocab_size: LoRAExtraVocabSize = Field(
51
+ default=256,
52
+ deprecated=(
53
+ "`lora_extra_vocab_size` is deprecated and will be removed "
54
+ "in v0.12.0. Additional vocabulary support for "
55
+ "LoRA adapters is being phased out."
56
+ ),
57
+ )
58
"""(Deprecated) Maximum size of extra vocabulary that can be present in a
59
LoRA adapter. Will be removed in v0.12.0."""
60
lora_vocab_padding_size: ClassVar[int] = (
@@ -60,7 +70,10 @@ class LoRAConfig:
70
per prompt. When run in offline mode, the lora IDs for n modalities
71
will be automatically assigned to 1-n with the names of the modalities
72
in alphabetic order."""
63
- bias_enabled: bool = False
73
+ bias_enabled: bool = Field(
74
+ default=False,
75
+ deprecated="`bias_enabled` is deprecated and will be removed in v0.12.0.",
76
+ )
77
"""[DEPRECATED] Enable bias for LoRA adapters. This option will be
78
removed in v0.12.0."""
79
@@ -87,36 +100,8 @@ class LoRAConfig:
100
hash_str = hashlib.md5(str(factors).encode(), usedforsecurity=False).hexdigest()
101
return hash_str
102
90
- def __post_init__(self):
91
- # Deprecation warning for lora_extra_vocab_size
92
- logger.warning(
93
- "`lora_extra_vocab_size` is deprecated and will be removed "
94
- "in v0.12.0. Additional vocabulary support for "
95
- "LoRA adapters is being phased out."
96
- )
97
-
98
- # Deprecation warning for enable_lora_bias
99
- if self.bias_enabled:
100
- logger.warning(
101
- "`enable_lora_bias` is deprecated and will be removed in v0.12.0."
102
- )
103
-
104
- # Setting the maximum rank to 512 should be able to satisfy the vast
105
- # majority of applications.
106
- possible_max_ranks = (1, 8, 16, 32, 64, 128, 256, 320, 512)
107
- possible_lora_extra_vocab_size = (256, 512)
108
- if self.max_lora_rank not in possible_max_ranks:
109
- raise ValueError(
110
- f"max_lora_rank ({self.max_lora_rank}) must be one of "
111
- f"{possible_max_ranks}."
112
- )
113
- if self.lora_extra_vocab_size not in possible_lora_extra_vocab_size:
114
- raise ValueError(
115
- f"lora_extra_vocab_size ({self.lora_extra_vocab_size}) "
116
- f"must be one of {possible_lora_extra_vocab_size}."
117
- )
118
- if self.max_loras < 1:
119
- raise ValueError(f"max_loras ({self.max_loras}) must be >= 1.")
103
+ @model_validator(mode="after")
104
+ def _validate_lora_config(self) -> Self:
105
if self.max_cpu_loras is None:
106
self.max_cpu_loras = self.max_loras
107
elif self.max_cpu_loras < self.max_loras:
@@ -125,6 +110,8 @@ class LoRAConfig:
110
f"max_loras ({self.max_loras})"
111
)
112
113
+ return self
114
+
115
def verify_with_cache_config(self, cache_config: CacheConfig):
116
if cache_config.cpu_offload_gb > 0 and not envs.VLLM_USE_V1:
117
raise ValueError("V0 LoRA does not support CPU offload, please use V1.")
vllm/config/utils.py
+8
@@ -11,6 +11,7 @@ from itertools import pairwise
11
from typing import TYPE_CHECKING, Any, Protocol, TypeVar
12
13
import regex as re
14
+from pydantic.fields import FieldInfo
15
from typing_extensions import runtime_checkable
16
17
if TYPE_CHECKING:
@@ -50,7 +51,14 @@ def get_field(cls: ConfigType, name: str) -> Field:
51
if (default_factory := named_field.default_factory) is not MISSING:
52
return field(default_factory=default_factory)
53
if (default := named_field.default) is not MISSING:
54
+ if isinstance(default, FieldInfo):
55
+ # Handle pydantic.Field defaults
56
+ if default.default_factory is not None:
57
+ return field(default_factory=default.default_factory)
58
+ else:
59
+ default = default.default
60
return field(default=default)
61
+
62
raise ValueError(
63
f"{cls.__name__}.{name} must have a default value or default factory."
64
)
vllm/engine/arg_utils.py
+1
-1
@@ -452,7 +452,7 @@ class EngineArgs:
452
num_gpu_blocks_override: Optional[int] = CacheConfig.num_gpu_blocks_override
453
num_lookahead_slots: int = SchedulerConfig.num_lookahead_slots
454
model_loader_extra_config: dict = get_field(LoadConfig, "model_loader_extra_config")
455
- ignore_patterns: Optional[Union[str, list[str]]] = LoadConfig.ignore_patterns
455
+ ignore_patterns: Union[str, list[str]] = get_field(LoadConfig, "ignore_patterns")
456
457
enable_chunked_prefill: Optional[bool] = SchedulerConfig.enable_chunked_prefill
458
disable_chunked_mm_input: bool = SchedulerConfig.disable_chunked_mm_input