A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 03:15 UTC
[Misc] Make `SchedulerConfig.max_model_len` init-only (#28733)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 15, 2025 at 17:59 UTC
638e4196d15f14a5fe68a64000801abda6c2ef8f
17 files changed
+22
-45
tests/kernels/moe/test_batched_moe.py
-2
@@ -40,8 +40,6 @@ NUM_EXPERTS = [8, 64]
40
TOP_KS = [1, 2, 6]
41
42
vllm_config = VllmConfig()
43
-vllm_config.scheduler_config.max_num_seqs = 128
44
-vllm_config.scheduler_config.max_model_len = 8192
43
44
45
@dataclass
tests/kernels/moe/test_block_fp8.py
-2
@@ -33,8 +33,6 @@ if current_platform.get_device_capability() < (9, 0):
33
pytest.skip("FP8 Triton requires CUDA 9.0 or higher", allow_module_level=True)
34
35
vllm_config = VllmConfig()
36
-vllm_config.scheduler_config.max_num_seqs = 128
37
-vllm_config.scheduler_config.max_model_len = 8192
36
37
# Test configurations
38
DTYPES = [torch.bfloat16] # [torch.half, torch.bfloat16, torch.float32]
tests/kernels/moe/test_block_int8.py
-2
@@ -18,8 +18,6 @@ if current_platform.get_device_capability() < (7, 0):
18
pytest.skip("INT8 Triton requires CUDA 7.0 or higher", allow_module_level=True)
19
20
vllm_config = VllmConfig()
21
-vllm_config.scheduler_config.max_num_seqs = 128
22
-vllm_config.scheduler_config.max_model_len = 8192
21
22
DTYPES = [torch.bfloat16]
23
tests/kernels/moe/test_cutlass_moe.py
-2
@@ -42,8 +42,6 @@ MNK_FACTORS = [
42
]
43
44
vllm_config = VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
45
-vllm_config.scheduler_config.max_num_seqs = 128
46
-vllm_config.scheduler_config.max_model_len = 8192
45
46
47
@dataclasses.dataclass
tests/kernels/moe/test_flashinfer.py
-2
@@ -45,8 +45,6 @@ MNK_FACTORS = [
45
]
46
47
vllm_config = VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
48
-vllm_config.scheduler_config.max_num_seqs = 128
49
-vllm_config.scheduler_config.max_model_len = 8192
48
49
50
def quant_fp8_per_tensor_batches(a):
tests/kernels/moe/test_moe.py
-2
@@ -81,8 +81,6 @@ FUSED_MOE_WN16_MNK_FACTORS = [
81
]
82
83
vllm_config = VllmConfig()
84
-vllm_config.scheduler_config.max_num_seqs = 128
85
-vllm_config.scheduler_config.max_model_len = 8192
84
85
86
def run_moe_test(
tests/kernels/moe/test_pplx_cutlass_moe.py
-2
@@ -192,8 +192,6 @@ def pplx_cutlass_moe(
192
193
194
vllm_config = VllmConfig()
195
-vllm_config.scheduler_config.max_num_seqs = 128
196
-vllm_config.scheduler_config.max_model_len = 8192
195
196
197
def _pplx_moe(
tests/kernels/moe/test_pplx_moe.py
-2
@@ -81,8 +81,6 @@ TOP_KS = [1, 2, 6]
81
DTYPES = [torch.float8_e4m3fn, torch.bfloat16]
82
83
vllm_config = VllmConfig()
84
-vllm_config.scheduler_config.max_num_seqs = 128
85
-vllm_config.scheduler_config.max_model_len = 8192
84
85
86
def torch_prepare(
tests/kernels/moe/test_triton_moe_ptpc_fp8.py
-2
@@ -18,8 +18,6 @@ if current_platform.get_device_capability() < (9, 0):
18
pytest.skip("FP8 Triton requires CUDA 9.0 or higher", allow_module_level=True)
19
20
vllm_config = VllmConfig()
21
-vllm_config.scheduler_config.max_num_seqs = 128
22
-vllm_config.scheduler_config.max_model_len = 8192
21
22
23
def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16):
tests/kernels/quantization/test_block_fp8.py
-2
@@ -29,8 +29,6 @@ if current_platform.get_device_capability() < (9, 0):
29
pytest.skip("FP8 Triton requires CUDA 9.0 or higher", allow_module_level=True)
30
31
vllm_config = VllmConfig()
32
-vllm_config.scheduler_config.max_num_seqs = 128
33
-vllm_config.scheduler_config.max_model_len = 8192
32
33
# Test configurations
34
DTYPES = [torch.bfloat16] # [torch.half, torch.bfloat16, torch.float32]
tests/kernels/quantization/test_block_int8.py
-2
@@ -18,8 +18,6 @@ if current_platform.get_device_capability() < (7, 0):
18
pytest.skip("INT8 Triton requires CUDA 7.0 or higher", allow_module_level=True)
19
20
vllm_config = VllmConfig()
21
-vllm_config.scheduler_config.max_num_seqs = 128
22
-vllm_config.scheduler_config.max_model_len = 8192
21
22
DTYPES = [torch.half, torch.bfloat16]
23
M = [1, 33, 64, 222]
vllm/config/scheduler.py
+18
-18
@@ -6,7 +6,7 @@ from collections.abc import Callable
6
from dataclasses import InitVar
7
from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast
8
9
-from pydantic import Field, field_validator, model_validator
9
+from pydantic import Field, field_validator
10
from pydantic.dataclasses import dataclass
11
from typing_extensions import Self, deprecated
12
@@ -48,13 +48,6 @@ class SchedulerConfig:
48
In real usage, this should be set in `EngineArgs.create_engine_config`.
49
"""
50
51
- max_model_len: int = Field(default=8192, ge=1)
52
- """Maximum length of a sequence (including prompt and generated text).
53
-
54
- The default value here is mainly for convenience when testing.
55
- In real usage, this should duplicate `ModelConfig.max_model_len` via
56
- `EngineArgs`."""
57
-
51
max_num_partial_prefills: int = Field(default=1, ge=1)
52
"""For chunked prefill, the maximum number of sequences that can be
53
partially prefilled concurrently."""
@@ -89,6 +82,12 @@ class SchedulerConfig:
82
is_multimodal_model: bool = False
83
"""True if the model is multimodal."""
84
85
+ max_model_len: InitVar[int] = 8192
86
+ """Maximum length of a sequence (including prompt and generated text).
87
+
88
+ Note: This is stored in the ModelConfig, and is used only here to
89
+ provide fallbacks and validate other attributes."""
90
+
91
is_encoder_decoder: InitVar[bool] = False
92
"""True if the model is an encoder-decoder model.
93
@@ -199,7 +198,7 @@ class SchedulerConfig:
198
return value
199
return handler(value)
200
202
- def __post_init__(self, is_encoder_decoder: bool) -> None:
201
+ def __post_init__(self, max_model_len: int, is_encoder_decoder: bool) -> None:
202
if is_encoder_decoder:
203
# Chunked prefill should be disabled for encoder-decoder models.
204
self.disable_chunked_mm_input = True
@@ -221,7 +220,7 @@ class SchedulerConfig:
220
221
if self.max_num_partial_prefills > 1:
222
if self.long_prefill_token_threshold == 0:
224
- self.long_prefill_token_threshold = int(self.max_model_len * 0.04)
223
+ self.long_prefill_token_threshold = int(max_model_len * 0.04)
224
225
logger.info(
226
"Concurrent partial prefills enabled with "
@@ -232,6 +231,8 @@ class SchedulerConfig:
231
self.long_prefill_token_threshold,
232
)
233
234
+ self.verify_max_model_len(max_model_len)
235
+
236
@property
237
@deprecated(
238
"`SchedulerConfig.chunked_prefill_enabled` has been renamed to "
@@ -245,15 +246,14 @@ class SchedulerConfig:
246
def chunked_prefill_enabled(self, value: bool):
247
self.enable_chunked_prefill = value
248
248
- @model_validator(mode="after")
249
- def _verify_args(self) -> Self:
249
+ def verify_max_model_len(self, max_model_len: int) -> Self:
250
if (
251
- self.max_num_batched_tokens < self.max_model_len
251
+ self.max_num_batched_tokens < max_model_len
252
and not self.enable_chunked_prefill
253
):
254
raise ValueError(
255
f"max_num_batched_tokens ({self.max_num_batched_tokens}) is "
256
- f"smaller than max_model_len ({self.max_model_len}). "
256
+ f"smaller than max_model_len ({max_model_len}). "
257
"This effectively limits the maximum sequence length to "
258
"max_num_batched_tokens and makes vLLM reject longer "
259
"sequences. Please increase max_num_batched_tokens or "
@@ -267,12 +267,12 @@ class SchedulerConfig:
267
f"({self.max_num_seqs})."
268
)
269
270
- if self.max_num_batched_tokens > self.max_num_seqs * self.max_model_len:
270
+ if self.max_num_batched_tokens > self.max_num_seqs * max_model_len:
271
logger.warning(
272
"max_num_batched_tokens (%d) exceeds max_num_seqs "
273
"* max_model_len (%d). This may lead to unexpected behavior.",
274
self.max_num_batched_tokens,
275
- self.max_num_seqs * self.max_model_len,
275
+ self.max_num_seqs * max_model_len,
276
)
277
278
if self.max_num_partial_prefills > 1:
@@ -282,11 +282,11 @@ class SchedulerConfig:
282
"max_num_partial_prefills > 1."
283
)
284
285
- if self.long_prefill_token_threshold > self.max_model_len:
285
+ if self.long_prefill_token_threshold > max_model_len:
286
raise ValueError(
287
"long_prefill_token_threshold "
288
f"({self.long_prefill_token_threshold}) cannot be greater "
289
- f"than the max_model_len ({self.max_model_len})."
289
+ f"than the max_model_len ({max_model_len})."
290
)
291
292
if self.max_long_partial_prefills > self.max_num_partial_prefills:
vllm/config/vllm.py
-1
@@ -929,7 +929,6 @@ class VllmConfig:
929
model_config = self.model_config
930
max_model_len = model_config.get_and_verify_max_len(max_model_len)
931
self.model_config.max_model_len = max_model_len
932
- self.scheduler_config.max_model_len = max_model_len
932
933
def try_verify_and_update_config(self):
934
if self.model_config is None:
vllm/platforms/cpu.py
+1
-1
@@ -339,7 +339,7 @@ class CpuPlatform(Platform):
339
)
340
vllm_config.scheduler_config.enable_chunked_prefill = False
341
vllm_config.scheduler_config.max_num_batched_tokens = max(
342
- vllm_config.scheduler_config.max_model_len,
342
+ vllm_config.model_config.max_model_len,
343
vllm_config.scheduler_config.DEFAULT_MAX_NUM_BATCHED_TOKENS,
344
)
345
vllm/platforms/tpu.py
+1
-1
@@ -191,7 +191,7 @@ class TpuPlatform(Platform):
191
)
192
vllm_config.scheduler_config.enable_chunked_prefill = False
193
vllm_config.scheduler_config.max_num_batched_tokens = max(
194
- vllm_config.scheduler_config.max_model_len,
194
+ vllm_config.model_config.max_model_len,
195
vllm_config.scheduler_config.DEFAULT_MAX_NUM_BATCHED_TOKENS,
196
)
197
vllm/platforms/xpu.py
+1
-1
@@ -185,7 +185,7 @@ class XPUPlatform(Platform):
185
)
186
vllm_config.scheduler_config.enable_chunked_prefill = False
187
vllm_config.scheduler_config.max_num_batched_tokens = max(
188
- vllm_config.scheduler_config.max_model_len,
188
+ vllm_config.model_config.max_model_len,
189
vllm_config.scheduler_config.DEFAULT_MAX_NUM_BATCHED_TOKENS,
190
)
191
vllm/v1/core/sched/scheduler.py
+1
-1
@@ -83,7 +83,7 @@ class Scheduler(SchedulerInterface):
83
# Scheduling constraints.
84
self.max_num_running_reqs = self.scheduler_config.max_num_seqs
85
self.max_num_scheduled_tokens = self.scheduler_config.max_num_batched_tokens
86
- self.max_model_len = self.scheduler_config.max_model_len
86
+ self.max_model_len = vllm_config.model_config.max_model_len
87
self.enable_kv_cache_events = (
88
self.kv_events_config is not None
89
and self.kv_events_config.enable_kv_cache_events