A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 06:00 UTC
Remove default values from `InitVar`s so that they're not stored (#29859)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Dec 2, 2025 at 12:16 UTC
951445a52df030050c9a3ed72d612d7e807ba368
17 files changed
+139
-77
benchmarks/benchmark_ngram_proposer.py
+4
-1
@@ -108,7 +108,10 @@ def benchmark_batched_propose(args):
108
device_config=DeviceConfig(device=current_platform.device_type),
109
parallel_config=ParallelConfig(),
110
load_config=LoadConfig(),
111
- scheduler_config=SchedulerConfig(),
111
+ scheduler_config=SchedulerConfig(
112
+ max_model_len=model_config.max_model_len,
113
+ is_encoder_decoder=model_config.is_encoder_decoder,
114
+ ),
115
)
116
117
# monkey patch vllm.v1.worker.gpu_model_runner.get_pp_group
tests/compile/test_fusion_attn.py
+10
-5
@@ -318,13 +318,18 @@ def test_attention_quant_pattern(
318
torch.set_default_dtype(dtype)
319
torch.manual_seed(42)
320
321
+ model_config = ModelConfig(
322
+ model=model_name,
323
+ max_model_len=2048,
324
+ dtype=dtype,
325
+ )
326
vllm_config = VllmConfig(
322
- model_config=ModelConfig(
323
- model=model_name,
324
- max_model_len=2048,
325
- dtype=dtype,
327
+ model_config=model_config,
328
+ scheduler_config=SchedulerConfig(
329
+ max_num_seqs=1024,
330
+ max_model_len=model_config.max_model_len,
331
+ is_encoder_decoder=model_config.is_encoder_decoder,
332
),
327
- scheduler_config=SchedulerConfig(max_num_seqs=1024),
333
compilation_config=CompilationConfig(
334
mode=CompilationMode.VLLM_COMPILE,
335
custom_ops=custom_ops_list,
tests/lora/test_worker.py
+17
-8
@@ -33,14 +33,16 @@ def test_worker_apply_lora(qwen3_lora_files):
33
lora_requests, lora_mapping
34
)
35
36
+ model_config = ModelConfig(
37
+ MODEL_PATH,
38
+ seed=0,
39
+ dtype="float16",
40
+ max_model_len=127,
41
+ enforce_eager=True,
42
+ )
43
+
44
vllm_config = VllmConfig(
37
- model_config=ModelConfig(
38
- MODEL_PATH,
39
- seed=0,
40
- dtype="float16",
41
- max_model_len=127,
42
- enforce_eager=True,
43
- ),
45
+ model_config=model_config,
46
load_config=LoadConfig(
47
download_dir=None,
48
load_format="dummy",
@@ -50,7 +52,14 @@ def test_worker_apply_lora(qwen3_lora_files):
52
tensor_parallel_size=1,
53
data_parallel_size=1,
54
),
53
- scheduler_config=SchedulerConfig("generate", 32, 32, 32),
55
+ scheduler_config=SchedulerConfig(
56
+ max_model_len=model_config.max_model_len,
57
+ is_encoder_decoder=model_config.is_encoder_decoder,
58
+ runner_type="generate",
59
+ max_num_batched_tokens=32,
60
+ max_num_seqs=32,
61
+ max_num_partial_prefills=32,
62
+ ),
63
device_config=DeviceConfig("cuda"),
64
cache_config=CacheConfig(
65
block_size=16,
tests/test_config.py
+13
@@ -6,12 +6,14 @@ from dataclasses import MISSING, Field, asdict, dataclass, field
6
from unittest.mock import patch
7
8
import pytest
9
+from pydantic import ValidationError
10
11
from vllm.compilation.backends import VllmBackend
12
from vllm.config import (
13
CompilationConfig,
14
ModelConfig,
15
PoolerConfig,
16
+ SchedulerConfig,
17
VllmConfig,
18
update_config,
19
)
@@ -1095,3 +1097,14 @@ def test_vllm_config_explicit_overrides():
1097
# Other fields should still use defaults
1098
assert config.compilation_config.mode == CompilationMode.VLLM_COMPILE
1099
assert config.compilation_config.cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
1100
+
1101
+
1102
+def test_scheduler_config_init():
1103
+ with pytest.raises(ValidationError):
1104
+ # Positional InitVars missing
1105
+ # (InitVars cannot have defaults otherwise they will become attributes)
1106
+ SchedulerConfig()
1107
+
1108
+ with pytest.raises(AttributeError):
1109
+ # InitVar does not become an attribute
1110
+ print(SchedulerConfig.default_factory().max_model_len)
tests/v1/attention/utils.py
+2
@@ -185,6 +185,8 @@ def create_vllm_config(
185
max_num_seqs=max_num_seqs,
186
max_num_batched_tokens=max_num_batched_tokens,
187
enable_chunked_prefill=enable_chunked_prefill,
188
+ max_model_len=model_config.max_model_len,
189
+ is_encoder_decoder=model_config.is_encoder_decoder,
190
)
191
192
device_config = DeviceConfig()
tests/v1/core/test_kv_cache_utils.py
+9
-2
@@ -1128,7 +1128,11 @@ def test_estimate_max_model_len(model_id, max_model_len, want_estimated_max_len)
1128
dtype="float16",
1129
max_model_len=max_model_len,
1130
)
1131
- scheduler_config = SchedulerConfig(max_num_batched_tokens=32768)
1131
+ scheduler_config = SchedulerConfig(
1132
+ max_num_batched_tokens=32768,
1133
+ max_model_len=model_config.max_model_len,
1134
+ is_encoder_decoder=model_config.is_encoder_decoder,
1135
+ )
1136
1137
vllm_config = VllmConfig(
1138
model_config=model_config,
@@ -1163,7 +1167,10 @@ def test_get_max_concurrency_for_kv_cache_config():
1167
max_model_len=max_model_len,
1168
)
1169
scheduler_config = SchedulerConfig(
1166
- max_num_batched_tokens=1024, enable_chunked_prefill=True
1170
+ max_num_batched_tokens=1024,
1171
+ enable_chunked_prefill=True,
1172
+ max_model_len=model_config.max_model_len,
1173
+ is_encoder_decoder=model_config.is_encoder_decoder,
1174
)
1175
1176
vllm_config = VllmConfig(
tests/v1/core/test_scheduler.py
+7
-6
@@ -1508,6 +1508,12 @@ def create_scheduler_with_priority(
1508
Returns:
1509
{class}`Scheduler` instance with priority scheduling
1510
"""
1511
+ model_config = ModelConfig(
1512
+ model=model,
1513
+ trust_remote_code=True,
1514
+ dtype="float16",
1515
+ seed=42,
1516
+ )
1517
if max_model_len is None:
1518
max_model_len = max_num_batched_tokens
1519
scheduler_config = SchedulerConfig(
@@ -1517,14 +1523,9 @@ def create_scheduler_with_priority(
1523
long_prefill_token_threshold=long_prefill_token_threshold,
1524
disable_chunked_mm_input=disable_chunked_mm_input,
1525
enable_chunked_prefill=True,
1526
+ is_encoder_decoder=model_config.is_encoder_decoder,
1527
policy="priority", # Enable priority scheduling
1528
)
1522
- model_config = ModelConfig(
1523
- model=model,
1524
- trust_remote_code=True,
1525
- dtype="float16",
1526
- seed=42,
1527
- )
1529
# Cache config, optionally force APC
1530
cache_config = CacheConfig(
1531
block_size=block_size,
tests/v1/core/utils.py
+8
-7
@@ -69,6 +69,13 @@ def create_scheduler(
69
Returns:
70
{class}`Scheduler` instance
71
"""
72
+ model_config = ModelConfig(
73
+ model=model,
74
+ trust_remote_code=True,
75
+ dtype="float16",
76
+ seed=42,
77
+ skip_tokenizer_init=skip_tokenizer_init,
78
+ )
79
if max_model_len is None:
80
max_model_len = max_num_batched_tokens
81
scheduler_config = SchedulerConfig(
@@ -79,13 +86,7 @@ def create_scheduler(
86
disable_chunked_mm_input=disable_chunked_mm_input,
87
enable_chunked_prefill=enable_chunked_prefill,
88
async_scheduling=async_scheduling,
82
- )
83
- model_config = ModelConfig(
84
- model=model,
85
- trust_remote_code=True,
86
- dtype="float16",
87
- seed=42,
88
- skip_tokenizer_init=skip_tokenizer_init,
89
+ is_encoder_decoder=model_config.is_encoder_decoder,
90
)
91
# Cache config, optionally force APC
92
cache_config = CacheConfig(
tests/v1/cudagraph/test_cudagraph_dispatch.py
+3
-1
@@ -40,7 +40,9 @@ def _create_vllm_config(
40
) -> MagicMock:
41
mock_config = MagicMock(spec=VllmConfig)
42
mock_config.compilation_config = compilation_config
43
- mock_config.scheduler_config = SchedulerConfig(max_num_seqs=max_num_seqs)
43
+ mock_config.scheduler_config = SchedulerConfig.default_factory(
44
+ max_num_seqs=max_num_seqs,
45
+ )
46
mock_config.parallel_config = ParallelConfig()
47
mock_config.speculative_config = None # No speculative decoding
48
if not lora_config:
tests/v1/engine/test_engine_core.py
+7
-6
@@ -484,12 +484,6 @@ def test_encoder_instance_zero_kv_cache(
484
vision encoder, so they don't need KV cache for text generation.
485
"""
486
# Form vllm config
487
- scheduler_config = SchedulerConfig(
488
- max_num_seqs=10,
489
- max_num_batched_tokens=512,
490
- max_model_len=512,
491
- disable_hybrid_kv_cache_manager=True,
492
- )
487
model_config = ModelConfig(
488
model="llava-hf/llava-1.5-7b-hf", # Multimodal model
489
enforce_eager=True,
@@ -497,6 +491,13 @@ def test_encoder_instance_zero_kv_cache(
491
dtype="float16",
492
seed=42,
493
)
494
+ scheduler_config = SchedulerConfig(
495
+ max_num_seqs=10,
496
+ max_num_batched_tokens=512,
497
+ max_model_len=512,
498
+ disable_hybrid_kv_cache_manager=True,
499
+ is_encoder_decoder=model_config.is_encoder_decoder,
500
+ )
501
cache_config = CacheConfig(
502
block_size=16,
503
gpu_memory_utilization=gpu_memory_utilization,
tests/v1/kv_connector/unit/utils.py
+7
-6
@@ -92,18 +92,19 @@ def create_vllm_config(
92
enable_permute_local_kv: bool = False,
93
) -> VllmConfig:
94
"""Initialize VllmConfig For Testing."""
95
- scheduler_config = SchedulerConfig(
96
- max_num_seqs=max_num_seqs,
97
- max_num_batched_tokens=max_num_batched_tokens,
98
- max_model_len=max_model_len,
99
- enable_chunked_prefill=enable_chunked_prefill,
100
- )
95
model_config = ModelConfig(
96
model=model,
97
trust_remote_code=True,
98
dtype="float16",
99
seed=42,
100
)
101
+ scheduler_config = SchedulerConfig(
102
+ max_num_seqs=max_num_seqs,
103
+ max_num_batched_tokens=max_num_batched_tokens,
104
+ max_model_len=max_model_len,
105
+ enable_chunked_prefill=enable_chunked_prefill,
106
+ is_encoder_decoder=model_config.is_encoder_decoder,
107
+ )
108
# Cache config, optionally force APC
109
cache_config = CacheConfig(
110
block_size=block_size,
tests/v1/spec_decode/test_eagle.py
+4
-1
@@ -66,7 +66,10 @@ def _create_proposer(
66
device_config=DeviceConfig(device=current_platform.device_type),
67
parallel_config=ParallelConfig(),
68
load_config=LoadConfig(),
69
- scheduler_config=SchedulerConfig(),
69
+ scheduler_config=SchedulerConfig(
70
+ max_model_len=model_config.max_model_len,
71
+ is_encoder_decoder=model_config.is_encoder_decoder,
72
+ ),
73
)
74
75
return EagleProposer(vllm_config=vllm_config, device=current_platform.device_type)
tests/v1/spec_decode/test_mtp.py
+4
-1
@@ -51,7 +51,10 @@ def _create_mtp_proposer(num_speculative_tokens: int) -> EagleProposer:
51
device_config=DeviceConfig(device=current_platform.device_type),
52
parallel_config=ParallelConfig(),
53
load_config=LoadConfig(),
54
- scheduler_config=SchedulerConfig(),
54
+ scheduler_config=SchedulerConfig(
55
+ max_model_len=model_config.max_model_len,
56
+ is_encoder_decoder=model_config.is_encoder_decoder,
57
+ ),
58
)
59
60
return EagleProposer(vllm_config=vllm_config, device=current_platform.device_type)
tests/v1/tpu/worker/test_tpu_model_runner.py
+6
-5
@@ -26,16 +26,17 @@ from vllm.v1.worker.tpu_model_runner import (
26
27
28
def get_vllm_config():
29
- scheduler_config = SchedulerConfig(
30
- max_num_seqs=10,
31
- max_num_batched_tokens=512,
32
- max_model_len=512,
33
- )
29
model_config = ModelConfig(
30
model="facebook/opt-125m",
31
dtype="bfloat16", # TPUs typically use bfloat16
32
seed=42,
33
)
34
+ scheduler_config = SchedulerConfig(
35
+ max_num_seqs=10,
36
+ max_num_batched_tokens=512,
37
+ max_model_len=512,
38
+ is_encoder_decoder=model_config.is_encoder_decoder,
39
+ )
40
cache_config = CacheConfig(
41
block_size=16,
42
gpu_memory_utilization=0.9,
tests/v1/worker/test_gpu_model_runner.py
+11
-9
@@ -79,16 +79,17 @@ def initialize_kv_cache(runner: GPUModelRunner):
79
80
81
def get_vllm_config():
82
- scheduler_config = SchedulerConfig(
83
- max_num_seqs=10,
84
- max_num_batched_tokens=512,
85
- max_model_len=512,
86
- )
82
model_config = ModelConfig(
83
model="facebook/opt-125m",
84
dtype="float16",
85
seed=42,
86
)
87
+ scheduler_config = SchedulerConfig(
88
+ max_num_seqs=10,
89
+ max_num_batched_tokens=512,
90
+ max_model_len=512,
91
+ is_encoder_decoder=model_config.is_encoder_decoder,
92
+ )
93
cache_config = CacheConfig(
94
block_size=BLOCK_SIZE,
95
gpu_memory_utilization=0.9,
@@ -784,14 +785,15 @@ def test_hybrid_attention_mamba_tensor_shapes(monkeypatch):
785
initialize_model_parallel(tensor_model_parallel_size=1)
786
torch.set_default_dtype(torch.float16)
787
788
+ model_config = ModelConfig(
789
+ model="ibm-granite/granite-4.0-tiny-preview",
790
+ dtype="float16",
791
+ )
792
scheduler_config = SchedulerConfig(
793
max_num_seqs=10,
794
max_num_batched_tokens=512,
795
max_model_len=512,
791
- )
792
- model_config = ModelConfig(
793
- model="ibm-granite/granite-4.0-tiny-preview",
794
- dtype="float16",
796
+ is_encoder_decoder=model_config.is_encoder_decoder,
797
)
798
cache_config = CacheConfig(
799
block_size=BLOCK_SIZE,
vllm/config/scheduler.py
+24
-18
@@ -28,6 +28,19 @@ SchedulerPolicy = Literal["fcfs", "priority"]
28
class SchedulerConfig:
29
"""Scheduler configuration."""
30
31
+ max_model_len: InitVar[int]
32
+ """Maximum length of a sequence (including prompt and generated text).
33
+
34
+ Note: This is stored in the ModelConfig, and is used only here to
35
+ provide fallbacks and validate other attributes."""
36
+
37
+ is_encoder_decoder: InitVar[bool]
38
+ """True if the model is an encoder-decoder model.
39
+
40
+ Note: This is stored in the ModelConfig, and is used only here to
41
+ disable chunked prefill and prefix caching for encoder-decoder models.
42
+ """
43
+
44
DEFAULT_MAX_NUM_BATCHED_TOKENS: ClassVar[int] = 2048
45
DEFAULT_MAX_NUM_SEQS: ClassVar[int] = 128
46
@@ -73,19 +86,6 @@ class SchedulerConfig:
86
is_multimodal_model: bool = False
87
"""True if the model is multimodal."""
88
76
- max_model_len: InitVar[int] = 8192
77
- """Maximum length of a sequence (including prompt and generated text).
78
-
79
- Note: This is stored in the ModelConfig, and is used only here to
80
- provide fallbacks and validate other attributes."""
81
-
82
- is_encoder_decoder: InitVar[bool] = False
83
- """True if the model is an encoder-decoder model.
84
-
85
- Note: This is stored in the ModelConfig, and is used only here to
86
- disable chunked prefill and prefix caching for encoder-decoder models.
87
- """
88
-
89
# TODO (ywang96): Make this configurable.
90
max_num_encoder_input_tokens: int = Field(init=False)
91
"""Multimodal encoder compute budget, only used in V1.
@@ -141,6 +141,17 @@ class SchedulerConfig:
141
while a larger value (e.g., 10) reduces host overhead and may increase throughput
142
by batching multiple tokens before sending."""
143
144
+ @staticmethod
145
+ def default_factory(**kwargs):
146
+ """
147
+ Factory method to create `SchedulerConfig` with default values for `InitVar`s.
148
+ """
149
+ if "max_model_len" not in kwargs:
150
+ kwargs["max_model_len"] = 8192
151
+ if "is_encoder_decoder" not in kwargs:
152
+ kwargs["is_encoder_decoder"] = False
153
+ return SchedulerConfig(**kwargs)
154
+
155
def get_scheduler_cls(self) -> type["SchedulerInterface"]:
156
if self.scheduler_cls is None:
157
if self.async_scheduling:
@@ -284,8 +295,3 @@ class SchedulerConfig:
295
)
296
297
return self
287
-
288
- def __getattribute__(self, name: str) -> Any:
289
- if name == "max_model_len" or name == "is_encoder_decoder":
290
- raise AttributeError(f"{name} is an init-only parameter. ")
291
- return object.__getattribute__(self, name)
vllm/config/vllm.py
+3
-1
@@ -170,7 +170,9 @@ class VllmConfig:
170
"""Cache configuration."""
171
parallel_config: ParallelConfig = Field(default_factory=ParallelConfig)
172
"""Parallel configuration."""
173
- scheduler_config: SchedulerConfig = Field(default_factory=SchedulerConfig)
173
+ scheduler_config: SchedulerConfig = Field(
174
+ default_factory=SchedulerConfig.default_factory,
175
+ )
176
"""Scheduler configuration."""
177
device_config: DeviceConfig = Field(default_factory=DeviceConfig)
178
"""Device configuration."""