A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 04:30 UTC
[Mypy] Fix mypy for `vllm/config` (#37808)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Mar 23, 2026 at 10:33 UTC
45bd5c8e758f8697fe40406f4650791b21c6baa4
15 files changed
+68
-59
tools/pre_commit/mypy.py
-1
@@ -40,7 +40,6 @@ EXCLUDE = [
40
"vllm/v1/attention/ops",
41
# TODO: Remove these entries after fixing mypy errors.
42
"vllm/benchmarks",
43
- "vllm/config",
43
]
44
45
vllm/config/attention.py
+1
-1
@@ -56,7 +56,7 @@ class AttentionConfig:
56
"""
57
from vllm.config.utils import get_hash_factors, hash_factors
58
59
- ignored_factors: list[str] = []
59
+ ignored_factors: set[str] = set()
60
factors = get_hash_factors(self, ignored_factors)
61
return hash_factors(factors)
62
vllm/config/compilation.py
+25
-21
@@ -116,29 +116,29 @@ class PassConfig:
116
"""
117
118
# New flags
119
- fuse_norm_quant: bool = Field(default=None)
119
+ fuse_norm_quant: bool | None = Field(default=None)
120
"""Fuse the custom RMSNorm + quant ops."""
121
- fuse_act_quant: bool = Field(default=None)
121
+ fuse_act_quant: bool | None = Field(default=None)
122
"""Fuse the custom SiluMul + quant ops."""
123
- fuse_attn_quant: bool = Field(default=None)
123
+ fuse_attn_quant: bool | None = Field(default=None)
124
"""Fuse the custom attention + quant ops."""
125
eliminate_noops: bool = Field(default=True)
126
"""Eliminate no-op ops."""
127
- enable_sp: bool = Field(default=None)
127
+ enable_sp: bool | None = Field(default=None)
128
"""Enable sequence parallelism. Requires TP>1. Automatically disabled
129
if the model's hidden_size is too small for SP to be beneficial
130
(threshold is device-capability dependent)."""
131
- fuse_gemm_comms: bool = Field(default=None)
131
+ fuse_gemm_comms: bool | None = Field(default=None)
132
"""Enable async TP."""
133
- fuse_allreduce_rms: bool = Field(default=None)
133
+ fuse_allreduce_rms: bool | None = Field(default=None)
134
"""Enable flashinfer allreduce fusion."""
135
enable_qk_norm_rope_fusion: bool = False
136
"""Enable fused Q/K RMSNorm + RoPE pass."""
137
138
# ROCm/AITER specific fusions
139
- fuse_act_padding: bool = Field(default=None)
139
+ fuse_act_padding: bool | None = Field(default=None)
140
"""Fuse the custom RMSNorm + padding ops."""
141
- fuse_rope_kvcache: bool = Field(default=None)
141
+ fuse_rope_kvcache: bool | None = Field(default=None)
142
"""Fuse the QK rope + KV cache ops."""
143
144
rope_kvcache_fusion_max_token_num: int = 256
@@ -198,9 +198,10 @@ class PassConfig:
198
199
if not current_platform.is_cuda():
200
return {}
201
- return FI_ALLREDUCE_FUSION_MAX_SIZE_MB.get(
202
- current_platform.get_device_capability().to_int(), {}
203
- )
201
+ capability = current_platform.get_device_capability()
202
+ if capability is None:
203
+ return {}
204
+ return FI_ALLREDUCE_FUSION_MAX_SIZE_MB.get(capability.to_int(), {})
205
206
def compute_hash(self) -> str:
207
"""
@@ -350,7 +351,7 @@ class DynamicShapesConfig:
351
352
from vllm.config.utils import get_hash_factors, hash_factors
353
353
- factors = get_hash_factors(self, {})
354
+ factors = get_hash_factors(self, set())
355
return hash_factors(factors)
356
357
@@ -404,7 +405,7 @@ class CompilationConfig:
405
"""
406
407
# Top-level Compilation control
407
- mode: CompilationMode = Field(default=None)
408
+ mode: CompilationMode = Field(default=None) # type: ignore[assignment]
409
"""The compilation approach used for torch.compile-based compilation of the
410
model.
411
@@ -544,7 +545,7 @@ class CompilationConfig:
545
constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`."""
546
547
# CudaGraph compilation
547
- cudagraph_mode: CUDAGraphMode = Field(default=None)
548
+ cudagraph_mode: CUDAGraphMode = Field(default=None) # type: ignore[assignment]
549
"""
550
The mode of the cudagraph:
551
@@ -606,7 +607,7 @@ class CompilationConfig:
607
When `enable_lora` is False, this option has no effect.
608
"""
609
609
- use_inductor_graph_partition: bool = Field(default=None)
610
+ use_inductor_graph_partition: bool = Field(default=None) # type: ignore[assignment]
611
"""Use inductor graph partition to split the graph at cudagraph_unsafe ops.
612
This partition happens at inductor codegen time after all passes and fusions
613
are finished. It generates a single `call` function which wraps
@@ -629,7 +630,7 @@ class CompilationConfig:
630
pass_config: PassConfig = field(default_factory=PassConfig)
631
"""Custom inductor passes, see PassConfig for more details"""
632
632
- max_cudagraph_capture_size: int = field(default=None)
633
+ max_cudagraph_capture_size: int | None = field(default=None)
634
"""The maximum cudagraph capture size.
635
636
If cudagraph_capture_sizes is specified, this will be set to the largest
@@ -769,7 +770,9 @@ class CompilationConfig:
770
exclude["pass_config"] = pass_config_exclude
771
772
config = TypeAdapter(CompilationConfig).dump_python(
772
- self, exclude=exclude, exclude_unset=True
773
+ self,
774
+ exclude=exclude, # type: ignore[arg-type]
775
+ exclude_unset=True,
776
)
777
778
return str(config)
@@ -991,7 +994,7 @@ class CompilationConfig:
994
- initialize compile_sizes
995
"""
996
994
- computed_compile_sizes = []
997
+ computed_compile_sizes: list[int] = []
998
if self.compile_sizes is not None:
999
# de-duplicate the sizes provided by the config
1000
self.compile_sizes = list(set(self.compile_sizes))
@@ -1001,6 +1004,7 @@ class CompilationConfig:
1004
"Unrecognized size type in compile_sizes, "
1005
f"expect 'cudagraph_capture_sizes', got {x}"
1006
)
1007
+ assert self.cudagraph_capture_sizes is not None
1008
computed_compile_sizes.extend(self.cudagraph_capture_sizes)
1009
else:
1010
assert isinstance(x, int)
@@ -1008,6 +1012,7 @@ class CompilationConfig:
1012
self.compile_sizes = computed_compile_sizes # type: ignore
1013
1014
# make sure the sizes are in ascending order
1015
+ assert self.cudagraph_capture_sizes is not None
1016
self.cudagraph_capture_sizes.sort()
1017
if self.cudagraph_capture_sizes:
1018
assert self.cudagraph_capture_sizes[-1] == self.max_cudagraph_capture_size
@@ -1099,6 +1104,7 @@ class CompilationConfig:
1104
1105
def set_splitting_ops_for_attn_fusion(self):
1106
assert self.pass_config.fuse_attn_quant
1107
+ assert self.cudagraph_mode is not None
1108
if self.splitting_ops is None:
1109
self.splitting_ops = []
1110
if self.cudagraph_mode.has_piecewise_cudagraphs():
@@ -1290,6 +1296,4 @@ class CompilationConfig:
1296
if self.compile_ranges_endpoints is None:
1297
return []
1298
endpoints = sorted(set(self.compile_ranges_endpoints))
1293
- return [
1294
- Range(start=s + 1, end=e) for s, e in zip([0] + endpoints[:-1], endpoints)
1295
- ]
1299
+ return [Range(s + 1, e) for s, e in zip([0] + endpoints[:-1], endpoints)]
vllm/config/device.py
+2
-2
@@ -13,8 +13,8 @@ from vllm.utils.hashing import safe_hash
13
Device = Literal["auto", "cuda", "cpu", "tpu", "xpu"]
14
15
16
-@config(config=ConfigDict(arbitrary_types_allowed=True))
17
-class DeviceConfig:
16
+@config(config=ConfigDict(arbitrary_types_allowed=True)) # type: ignore[arg-type,misc]
17
+class DeviceConfig: # type: ignore[misc]
18
"""Configuration for the device to use for vLLM execution."""
19
20
device: SkipValidation[Device | torch.device | None] = "auto"
vllm/config/kernel.py
+1
-1
@@ -26,7 +26,7 @@ MoEBackend = Literal[
26
class KernelConfig:
27
"""Configuration for kernel selection and warmup behavior."""
28
29
- enable_flashinfer_autotune: bool = Field(default=None)
29
+ enable_flashinfer_autotune: bool | None = Field(default=None)
30
"""If True, run FlashInfer autotuning during kernel warmup."""
31
32
moe_backend: MoEBackend = "auto"
vllm/config/kv_events.py
+1
-1
@@ -18,7 +18,7 @@ class KVEventsConfig:
18
Events can be published externally by zmq using the event publisher config.
19
"""
20
21
- publisher: Literal["null", "zmq"] = Field(default=None)
21
+ publisher: Literal["null", "zmq"] | None = Field(default=None)
22
"""The publisher to use for publishing kv events. Can be "null", "zmq".
23
"""
24
vllm/config/lora.py
+2
-2
@@ -25,8 +25,8 @@ MaxLoRARanks = Literal[1, 8, 16, 32, 64, 128, 256, 320, 512]
25
LoRAExtraVocabSize = Literal[256, 512]
26
27
28
-@config(config=ConfigDict(arbitrary_types_allowed=True))
29
-class LoRAConfig:
28
+@config(config=ConfigDict(arbitrary_types_allowed=True)) # type: ignore[arg-type,misc]
29
+class LoRAConfig: # type: ignore[misc]
30
"""Configuration for LoRA."""
31
32
max_lora_rank: MaxLoRARanks = 16
vllm/config/model.py
+11
-10
@@ -93,7 +93,7 @@ LayerBlockType = Literal["attention", "linear_attention", "mamba"]
93
94
_RUNNER_CONVERTS: dict[RunnerType, list[ConvertType]] = {
95
"generate": [],
96
- "pooling": ["embed", "classify", "reward"],
96
+ "pooling": ["embed", "classify"],
97
"draft": [],
98
}
99
@@ -102,8 +102,8 @@ AttnTypeStr = Literal[
102
]
103
104
105
-@config(config=ConfigDict(arbitrary_types_allowed=True))
106
-class ModelConfig:
105
+@config(config=ConfigDict(arbitrary_types_allowed=True)) # type: ignore[arg-type,misc]
106
+class ModelConfig: # type: ignore[misc]
107
"""Configuration for the model."""
108
109
model: str = "Qwen/Qwen3-0.6B"
@@ -121,7 +121,7 @@ class ModelConfig:
121
"""Convert the model using adapters defined in
122
[vllm.model_executor.models.adapters][]. The most common use case is to
123
adapt a text generation model to be used for pooling tasks."""
124
- tokenizer: str = Field(default=None)
124
+ tokenizer: str = Field(default=None) # type: ignore[assignment]
125
"""Name or path of the Hugging Face tokenizer to use. If unspecified, model
126
name or path will be used."""
127
tokenizer_mode: TokenizerMode | str = "auto"
@@ -177,7 +177,7 @@ class ModelConfig:
177
"""The specific revision to use for the tokenizer on the Hugging Face Hub.
178
It can be a branch name, a tag name, or a commit id. If unspecified, will
179
use the default version."""
180
- max_model_len: int = Field(default=None, ge=-1)
180
+ max_model_len: int = Field(default=None, ge=-1) # type: ignore[assignment]
181
"""Model context length (prompt and output). If unspecified, will be
182
automatically derived from the model config.
183
@@ -454,7 +454,7 @@ class ModelConfig:
454
self.hf_config_path = maybe_model_redirect(self.hf_config_path)
455
456
if callable(self.hf_overrides):
457
- hf_overrides_kw = {}
457
+ hf_overrides_kw: dict[str, Any] = {}
458
hf_overrides_fn = self.hf_overrides
459
dict_overrides: dict[str, Any] = {}
460
else:
@@ -582,7 +582,7 @@ class ModelConfig:
582
self.dtype,
583
is_pooling_model=self.runner_type == "pooling",
584
revision=self.revision,
585
- config_format=self.config_format,
585
+ config_format=self.config_format, # type: ignore[arg-type]
586
)
587
588
self.original_max_model_len = self.max_model_len
@@ -626,7 +626,7 @@ class ModelConfig:
626
k: v for k, v in mm_config_kwargs.items() if v is not None
627
}
628
629
- self.multimodal_config = MultiModalConfig(**mm_config_kwargs)
629
+ self.multimodal_config = MultiModalConfig(**mm_config_kwargs) # type: ignore[arg-type]
630
631
# Multimodal GGUF models must use original repo for mm processing
632
if is_gguf(self.tokenizer) and self.is_multimodal_model:
@@ -732,7 +732,7 @@ class ModelConfig:
732
733
@property
734
def architectures(self) -> list[str]:
735
- return self.model_arch_config.architectures
735
+ return self.model_arch_config.architectures # type: ignore[return-value]
736
737
@property
738
def architecture(self) -> str:
@@ -1004,7 +1004,7 @@ class ModelConfig:
1004
is_bitsandbytes = self.quantization == "bitsandbytes"
1005
has_quantization_config = self.model_arch_config.quantization_config is not None
1006
is_8bit = (
1007
- self.model_arch_config.quantization_config.get("load_in_8bit", False)
1007
+ self.model_arch_config.quantization_config.get("load_in_8bit", False) # type: ignore[union-attr]
1008
if has_quantization_config
1009
else False
1010
)
@@ -1292,6 +1292,7 @@ class ModelConfig:
1292
"attn_type_list, or a layer_types in the hf_config, "
1293
f"cannot determine the num of {block_type} layers"
1294
)
1295
+ raise AssertionError(f"Unsupported block type: {block_type}")
1296
1297
def get_mamba_chunk_size(self) -> int | None:
1298
"""
vllm/config/pooler.py
+2
-2
@@ -108,14 +108,14 @@ class PoolerConfig:
108
pooling_type,
109
pooling_type,
110
)
111
- self.seq_pooling_type = pooling_type
111
+ self.seq_pooling_type = pooling_type # type: ignore[assignment]
112
elif pooling_type in TOK_POOLING_TYPES:
113
logger.debug(
114
"Resolved `pooling_type=%r` to `tok_pooling_type=%r`.",
115
pooling_type,
116
pooling_type,
117
)
118
- self.tok_pooling_type = pooling_type
118
+ self.tok_pooling_type = pooling_type # type: ignore[assignment]
119
else:
120
raise NotImplementedError(pooling_type)
121
vllm/config/scheduler.py
+1
-1
@@ -173,7 +173,7 @@ class SchedulerConfig:
173
logger.warning_once(
174
"Using custom scheduler class %s. This scheduler interface is "
175
"not public and compatibility may not be maintained.",
176
- self.scheduler_cls,
176
+ self.scheduler_cls, # type: ignore[arg-type]
177
)
178
if not isinstance(self.scheduler_cls, str):
179
return cast(type["SchedulerInterface"], self.scheduler_cls)
vllm/config/speculative.py
+2
-2
@@ -67,7 +67,7 @@ class SpeculativeConfig:
67
enforce_eager: bool | None = None
68
"""Override the default enforce_eager from model_config"""
69
# General speculative decoding control
70
- num_speculative_tokens: int = Field(default=None, gt=0)
70
+ num_speculative_tokens: int = Field(default=None, gt=0) # type: ignore[assignment]
71
"""The number of speculative tokens, if provided. It will default to the
72
number in the draft model config if present, otherwise, it is required."""
73
model: str | None = None
@@ -89,7 +89,7 @@ class SpeculativeConfig:
89
warn users when they mistakenly provide the wrong argument."""
90
91
# Draft model configuration
92
- quantization: me_quant.QuantizationMethods | None = None
92
+ quantization: me_quant.QuantizationMethods | str | None = None
93
"""Quantization method that was used to quantize the draft model weights.
94
If `None`, we assume the model weights are not quantized. Note that it only
95
takes effect when using the draft model-based speculative method."""
vllm/config/utils.py
+4
-4
@@ -11,13 +11,13 @@ import os
11
import pathlib
12
import textwrap
13
from collections.abc import Callable, Mapping, Sequence, Set
14
-from dataclasses import MISSING, field, fields, is_dataclass
14
+from dataclasses import MISSING, dataclass, field, fields, is_dataclass
15
from itertools import pairwise
16
from typing import TYPE_CHECKING, Any, Protocol, TypeVar, cast
17
18
import torch
19
from pydantic import ConfigDict
20
-from pydantic.dataclasses import dataclass
20
+from pydantic.dataclasses import dataclass as pydantic_dataclass
21
from pydantic.fields import Field as PydanticField
22
from pydantic.fields import FieldInfo
23
from typing_extensions import dataclass_transform, runtime_checkable
@@ -58,8 +58,8 @@ def config(
58
if config is not None:
59
merged_config.update(config)
60
61
- def decorator(cls):
62
- return dataclass(cls, config=merged_config, **kwargs)
61
+ def decorator(cls: type[ConfigT]) -> type[ConfigT]:
62
+ return pydantic_dataclass(cls, config=merged_config, **kwargs) # type: ignore[return-value]
63
64
# Called with arguments: @config(config=...)
65
if cls is None:
vllm/config/vllm.py
+8
-8
@@ -243,15 +243,15 @@ OPTIMIZATION_LEVEL_TO_CONFIG = {
243
}
244
245
246
-@config(config=ConfigDict(arbitrary_types_allowed=True))
247
-class VllmConfig:
246
+@config(config=ConfigDict(arbitrary_types_allowed=True)) # type: ignore[arg-type,misc]
247
+class VllmConfig: # type: ignore[misc]
248
"""Dataclass which contains all vllm-related configuration. This
249
simplifies passing around the distinct configurations in the codebase.
250
"""
251
252
# TODO: use default_factory once default constructing ModelConfig doesn't
253
# try to download a model
254
- model_config: ModelConfig = Field(default=None)
254
+ model_config: ModelConfig = Field(default=None) # type: ignore[assignment]
255
"""Model configuration."""
256
cache_config: CacheConfig = Field(default_factory=CacheConfig)
257
"""Cache configuration."""
@@ -883,7 +883,7 @@ class VllmConfig:
883
884
tp_size = self.parallel_config.tensor_parallel_size
885
hidden_size = self.model_config.get_hidden_size()
886
- element_size = self.model_config.dtype.itemsize
886
+ element_size = self.model_config.dtype.itemsize # type: ignore[union-attr]
887
pass_config.sp_min_token_num = get_sequence_parallelism_threshold(
888
hidden_size, tp_size, element_size
889
)
@@ -1061,7 +1061,7 @@ class VllmConfig:
1061
1062
is_fullgraph = (
1063
self.compilation_config.use_inductor_graph_partition
1064
- or len(self.compilation_config.splitting_ops) == 0
1064
+ or len(self.compilation_config.splitting_ops or []) == 0
1065
)
1066
if self.parallel_config.pipeline_parallel_size > 1 or not is_fullgraph:
1067
if "-rms_norm" not in self.compilation_config.custom_ops:
@@ -1216,7 +1216,7 @@ class VllmConfig:
1216
)
1217
self.compilation_config.debug_dump_path = env_path
1218
1219
- def has_blocked_weights():
1219
+ def has_blocked_weights(): # type: ignore[no-redef]
1220
if self.quant_config is not None:
1221
if hasattr(self.quant_config, "weight_block_size"):
1222
return self.quant_config.weight_block_size is not None
@@ -1474,7 +1474,7 @@ class VllmConfig:
1474
if max_size is not None:
1475
max_token_num = max_size // (
1476
self.model_config.get_hidden_size()
1477
- * self.model_config.dtype.itemsize
1477
+ * self.model_config.dtype.itemsize # type: ignore[union-attr]
1478
)
1479
if compile_range_end is not None and max_token_num < compile_range_end:
1480
computed_compile_ranges_endpoints.append(max_token_num)
@@ -1497,7 +1497,7 @@ class VllmConfig:
1497
1498
tp_size = self.parallel_config.tensor_parallel_size
1499
hidden_size = self.model_config.get_hidden_size()
1500
- element_size = self.model_config.dtype.itemsize
1500
+ element_size = self.model_config.dtype.itemsize # type: ignore[union-attr]
1501
pass_config.sp_min_token_num = get_sequence_parallelism_threshold(
1502
hidden_size, tp_size, element_size
1503
)
vllm/engine/arg_utils.py
+1
-1
@@ -1924,7 +1924,7 @@ class EngineArgs:
1924
)
1925
1926
offload_config = OffloadConfig(
1927
- offload_backend=self.offload_backend,
1927
+ offload_backend=self.offload_backend, # type: ignore[arg-type]
1928
uva=UVAOffloadConfig(
1929
cpu_offload_gb=self.cpu_offload_gb,
1930
cpu_offload_params=self.cpu_offload_params,
vllm/v1/cudagraph_dispatcher.py
+7
-2
@@ -72,6 +72,9 @@ class CudagraphDispatcher:
72
"""Pre-compute the mapping from batch size to padded graph size."""
73
max_size = self.compilation_config.max_cudagraph_capture_size
74
capture_sizes = self.compilation_config.cudagraph_capture_sizes
75
+ assert max_size is not None, (
76
+ "Maximum cudagraph capture size must be set when cudagraphs are enabled."
77
+ )
78
assert capture_sizes is not None, (
79
"Cudagraph capture sizes must be set when cudagraphs are enabled."
80
)
@@ -94,7 +97,7 @@ class CudagraphDispatcher:
97
):
98
for size in self.compilation_config.compile_sizes:
99
size = int(size)
97
- if size <= self.compilation_config.max_cudagraph_capture_size:
100
+ if size <= max_size:
101
padded = self._bs_to_padded_graph_size[size]
102
if padded != size:
103
raise ValueError(
@@ -265,11 +268,13 @@ class CudagraphDispatcher:
268
f"No allowed cudagraph modes: valid_modes={valid_modes}, "
269
f"invalid_modes={invalid_modes}"
270
)
271
+ max_size = self.compilation_config.max_cudagraph_capture_size
272
273
if (
274
not self.keys_initialized
275
or self.cudagraph_mode == CUDAGraphMode.NONE
272
- or num_tokens > self.compilation_config.max_cudagraph_capture_size
276
+ or max_size is None
277
+ or num_tokens > max_size
278
or allowed_modes <= {CUDAGraphMode.NONE}
279
):
280
return CUDAGraphMode.NONE, BatchDescriptor(num_tokens)