A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[V0 Deprecation] Remove max_seq_len_to_capture (#25543)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Sep 24, 2025 at 01:51 UTC
2e19a848d42d95ebef0a5760f9cff673acaf5ca1
7 files changed
+2
-48
tests/tpu/lora/test_lora.py
-1
@@ -31,7 +31,6 @@ def use_v1_only(monkeypatch: pytest.MonkeyPatch):
31
def setup_vllm(num_loras: int, tp: int) -> vllm.LLM:
32
return vllm.LLM(model="Qwen/Qwen2.5-3B-Instruct",
33
max_model_len=256,
34
- max_seq_len_to_capture=256,
34
max_num_seqs=8,
35
tensor_parallel_size=tp,
36
enable_lora=True,
vllm/attention/backends/utils.py
+2
-2
@@ -304,7 +304,7 @@ class CommonAttentionState(AttentionState):
304
max_query_len=1,
305
max_decode_query_len=1,
306
max_prefill_seq_len=0,
307
- max_decode_seq_len=self.runner.max_seq_len_to_capture,
307
+ max_decode_seq_len=self.runner.max_model_len,
308
query_start_loc=None,
309
seq_start_loc=None,
310
context_lens_tensor=None,
@@ -390,7 +390,7 @@ class CommonAttentionState(AttentionState):
390
dtype=torch.int).cuda()
391
attn_metadata.encoder_seq_lens_tensor = torch.full(
392
(batch_size, ), 1, dtype=torch.int).cuda()
393
- attn_metadata.max_encoder_seq_len = self.runner.max_seq_len_to_capture
393
+ attn_metadata.max_encoder_seq_len = self.runner.max_model_len
394
attn_metadata.num_encoder_tokens = 0
395
396
def _add_additional_input_buffers_for_enc_dec_model(
vllm/config/model.py
-18
@@ -177,11 +177,6 @@ class ModelConfig:
177
graph and always execute the model in eager mode. If False, we will use
178
CUDA graph and eager execution in hybrid for maximal performance and
179
flexibility."""
180
- max_seq_len_to_capture: int = 8192
181
- """Maximum sequence len covered by CUDA graphs. When a sequence has context
182
- length larger than this, we fall back to eager mode. Additionally for
183
- encoder-decoder models, if the sequence length of the encoder input is
184
- larger than this, we fall back to the eager mode."""
180
max_logprobs: int = 20
181
"""Maximum number of log probabilities to return when `logprobs` is
182
specified in `SamplingParams`. The default value comes the default for the
@@ -1024,21 +1019,8 @@ class ModelConfig:
1019
current_platform.verify_quantization(self.quantization)
1020
1021
def _verify_cuda_graph(self) -> None:
1027
- # The `max_seq_len_to_capture` was incorrectly
1028
- # based on the encoder's input length (448)
1029
- # but not the decoder's larger input length (1500).
1030
- # This change ensures the CUDA Graph captures the correct,
1031
- # larger sequence length, allowing it to work as intended.
1032
- effective_max_seq_len = self.max_model_len
1033
- if self.is_encoder_decoder:
1034
- effective_max_seq_len = max(
1035
- effective_max_seq_len,
1036
- getattr(self.hf_config, "max_source_positions", 0))
1037
- self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
1038
- effective_max_seq_len)
1022
# CUDAGraph capture not supported for encoder-decoder models on ROCm
1023
unsupported_rocm = self.is_encoder_decoder
1041
-
1024
if (unsupported_rocm and not self.enforce_eager
1025
and current_platform.is_rocm()):
1026
logger.warning(
vllm/config/speculative.py
-2
@@ -285,8 +285,6 @@ class SpeculativeConfig:
285
max_model_len,
286
quantization=self.quantization,
287
enforce_eager=self.target_model_config.enforce_eager,
288
- max_seq_len_to_capture=self.target_model_config.
289
- max_seq_len_to_capture,
288
max_logprobs=self.target_model_config.max_logprobs,
289
hf_overrides=SpeculativeConfig.hf_config_override,
290
)
vllm/engine/arg_utils.py
-4
@@ -373,7 +373,6 @@ class EngineArgs:
373
tokenizer_revision: Optional[str] = ModelConfig.tokenizer_revision
374
quantization: Optional[QuantizationMethods] = ModelConfig.quantization
375
enforce_eager: bool = ModelConfig.enforce_eager
376
- max_seq_len_to_capture: int = ModelConfig.max_seq_len_to_capture
376
disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
377
limit_mm_per_prompt: dict[str, int] = \
378
get_field(MultiModalConfig, "limit_per_prompt")
@@ -545,8 +544,6 @@ class EngineArgs:
544
**model_kwargs["quantization"])
545
model_group.add_argument("--enforce-eager",
546
**model_kwargs["enforce_eager"])
548
- model_group.add_argument("--max-seq-len-to-capture",
549
- **model_kwargs["max_seq_len_to_capture"])
547
model_group.add_argument("--max-logprobs",
548
**model_kwargs["max_logprobs"])
549
model_group.add_argument("--logprobs-mode",
@@ -1008,7 +1005,6 @@ class EngineArgs:
1005
max_model_len=self.max_model_len,
1006
quantization=self.quantization,
1007
enforce_eager=self.enforce_eager,
1011
- max_seq_len_to_capture=self.max_seq_len_to_capture,
1008
max_logprobs=self.max_logprobs,
1009
logprobs_mode=self.logprobs_mode,
1010
disable_sliding_window=self.disable_sliding_window,
vllm/entrypoints/llm.py
-7
@@ -130,11 +130,6 @@ class LLM:
130
enforce_eager: Whether to enforce eager execution. If True, we will
131
disable CUDA graph and always execute the model in eager mode.
132
If False, we will use CUDA graph and eager execution in hybrid.
133
- max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
134
- When a sequence has context length larger than this, we fall back
135
- to eager mode. Additionally for encoder-decoder models, if the
136
- sequence length of the encoder input is larger than this, we fall
137
- back to the eager mode.
133
disable_custom_all_reduce: See
134
[ParallelConfig][vllm.config.ParallelConfig].
135
hf_token: The token to use as HTTP bearer authorization for remote files
@@ -184,7 +179,6 @@ class LLM:
179
swap_space: float = 4,
180
cpu_offload_gb: float = 0,
181
enforce_eager: bool = False,
187
- max_seq_len_to_capture: int = 8192,
182
disable_custom_all_reduce: bool = False,
183
hf_token: Optional[Union[bool, str]] = None,
184
hf_overrides: Optional[HfOverrides] = None,
@@ -281,7 +275,6 @@ class LLM:
275
swap_space=swap_space,
276
cpu_offload_gb=cpu_offload_gb,
277
enforce_eager=enforce_eager,
284
- max_seq_len_to_capture=max_seq_len_to_capture,
278
disable_custom_all_reduce=disable_custom_all_reduce,
279
hf_token=hf_token,
280
hf_overrides=hf_overrides,
vllm/model_executor/models/config.py
-14
@@ -245,19 +245,6 @@ class SnowflakeGteNewModelConfig(VerifyAndUpdateConfig):
245
}
246
247
248
-class GraniteMoeHybridModelConfig(VerifyAndUpdateConfig):
249
-
250
- @staticmethod
251
- def verify_and_update_config(vllm_config: "VllmConfig") -> None:
252
- config = vllm_config.model_config
253
- config.max_seq_len_to_capture = config.max_model_len
254
- logger.info(
255
- "Setting max_seq_len_to_capture to %d "
256
- "to ensure that CUDA graph capture "
257
- "covers sequences of length up to max_model_len.",
258
- config.max_model_len)
259
-
260
-
248
class GptOssForCausalLMConfig(VerifyAndUpdateConfig):
249
250
@staticmethod
@@ -426,7 +413,6 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
413
"XLMRobertaModel": JinaRobertaModelConfig,
414
"JinaVLForRanking": JinaVLForSequenceClassificationConfig,
415
"JambaForSequenceClassification": JambaForSequenceClassificationConfig,
429
- "GraniteMoeHybridForCausalLM": GraniteMoeHybridModelConfig,
416
"GptOssForCausalLM": GptOssForCausalLMConfig,
417
"MambaForCausalLM": MambaModelConfig,
418
"Mamba2ForCausalLM": MambaModelConfig,