A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
[BugFix] Fix `VllmConfig()` construction on all platforms (#20695)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Jul 10, 2025 at 08:00 UTC
ffbcc9e757c9feafa6d72c05fca2fa9eafffecdd
5 files changed
+19
-16
vllm/config.py
-1
@@ -4722,7 +4722,6 @@ class VllmConfig:
4722
# calculate the default `batch_size_capture_list`
4723
if not envs.VLLM_USE_V1:
4724
batch_size_capture_list = []
4725
- max_batchsize_to_capture = 0
4725
if self.scheduler_config is not None and \
4726
self.model_config is not None and \
4727
not self.model_config.enforce_eager:
vllm/platforms/cpu.py
+4
-3
@@ -96,7 +96,8 @@ class CpuPlatform(Platform):
96
from vllm.utils import GiB_bytes
97
model_config = vllm_config.model_config
98
99
- model_config.disable_cascade_attn = True
99
+ if model_config is not None:
100
+ model_config.disable_cascade_attn = True
101
102
cache_config = vllm_config.cache_config
103
@@ -123,7 +124,7 @@ class CpuPlatform(Platform):
124
"CPU backend doesn't support fp8_e4m3 KV cache type, "
125
"cast to fp8_e5m2.")
126
126
- if (cache_config.cache_dtype != "auto"
127
+ if (cache_config.cache_dtype != "auto" and model_config is not None
128
and model_config.dtype == torch.half):
129
logger.warning("FP8 KV cache on the CPU backend only does not"
130
" support fp16 for now, cast to bf16.")
@@ -229,7 +230,7 @@ class CpuPlatform(Platform):
230
os.environ["LOCAL_WORLD_SIZE"] = str(
231
vllm_config.parallel_config.tensor_parallel_size)
232
232
- if vllm_config.model_config and vllm_config.model_config.use_mla:
233
+ if model_config is not None and model_config.use_mla:
234
logger.info(
235
"MLA is enabled on a non-GPU platform; forcing chunked "
236
"prefill and prefix caching to be disabled.")
vllm/platforms/cuda.py
+5
-3
@@ -166,17 +166,19 @@ class CudaPlatformBase(Platform):
166
logger.info(
167
"Forcing kv cache block size to 64 for FlashMLA backend.")
168
169
+ compilation_config = vllm_config.compilation_config
170
if (envs.VLLM_ALL2ALL_BACKEND == "deepep_high_throughput"
171
and parallel_config.data_parallel_size > 1
171
- and vllm_config.compilation_config.use_cudagraph):
172
+ and compilation_config.use_cudagraph):
173
logger.info(
174
"Data Parallel: Forcing enforce eager to be True since DP "
175
"with DeepEP high-throughput kernels are not CUDA Graph "
176
"compatible. The DeepEP low-latency kernels are CUDA Graph "
177
"compatible. Set the all_to_all backend to deepep_low_latency "
178
"to use those kernels instead.")
178
- vllm_config.compilation_config.use_cudagraph = False
179
- vllm_config.model_config.enforce_eager = True
179
+ compilation_config.use_cudagraph = False
180
+ if model_config is not None:
181
+ model_config.enforce_eager = True
182
# TODO (varun): Turning this ON gives incorrect results for the
183
# Deepseek-V2-lite model.
184
vllm_config.compilation_config.use_inductor = False
vllm/platforms/tpu.py
+6
-4
@@ -116,11 +116,13 @@ class TpuPlatform(Platform):
116
assert vllm_config.speculative_config is None, \
117
"TPU does not support speculative decoding"
118
119
- if vllm_config.model_config.dtype in (torch.float16, torch.float32):
119
+ model_config = vllm_config.model_config
120
+ if model_config is not None and model_config.dtype in (torch.float16,
121
+ torch.float32):
122
logger.warning(
123
"The TPU backend currently does not support %s. "
122
- "Using bfloat16 instead.", vllm_config.model_config.dtype)
123
- vllm_config.model_config.dtype = torch.bfloat16
124
+ "Using bfloat16 instead.", model_config.dtype)
125
+ model_config.dtype = torch.bfloat16
126
127
from vllm.v1.attention.backends.pallas import PallasAttentionBackend
128
cache_config.block_size = PallasAttentionBackend.get_page_size(
@@ -146,7 +148,7 @@ class TpuPlatform(Platform):
148
"Forcing --disable_chunked_mm_input.")
149
scheduler_config.disable_chunked_mm_input = True
150
149
- if vllm_config.model_config and vllm_config.model_config.use_mla:
151
+ if model_config and model_config.use_mla:
152
logger.info(
153
"MLA is enabled on a non-GPU platform; forcing chunked "
154
"prefill and prefix caching to be disabled.")
vllm/platforms/xpu.py
+4
-5
@@ -85,14 +85,14 @@ class XPUPlatform(Platform):
85
@classmethod
86
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
87
cache_config = vllm_config.cache_config
88
+ model_config = vllm_config.model_config
89
# in V1(or with ipex chunked prefill) block_size is 64
90
if cache_config and cache_config.block_size is None:
91
cache_config.block_size = 64
92
93
# FIXME: Temporarily forcing eager mode
94
# remove after t.compile support stabilizes.
94
-
95
- if (envs.VLLM_USE_V1 and vllm_config.model_config is not None
95
+ if (envs.VLLM_USE_V1 and model_config is not None
96
and not vllm_config.model_config.enforce_eager):
97
from vllm.config import CompilationLevel
98
vllm_config.compilation_config.level = CompilationLevel.NO_COMPILATION # noqa: E501
@@ -100,8 +100,7 @@ class XPUPlatform(Platform):
100
# Instances created using VllmConfig() typically have model_config as
101
# None by default. The modification involves adding a check to prevent
102
# potential null exceptions check and update model config.
103
- if vllm_config.model_config is not None:
104
- model_config = vllm_config.model_config
103
+ if model_config is not None:
104
if model_config.dtype == torch.bfloat16:
105
bf16_supported = cls.device_support_bf16()
106
if not bf16_supported:
@@ -139,7 +138,7 @@ class XPUPlatform(Platform):
138
parallel_config.distributed_executor_backend)
139
parallel_config.distributed_executor_backend = "ray"
140
142
- if vllm_config.model_config and vllm_config.model_config.use_mla:
141
+ if model_config and model_config.use_mla:
142
logger.info(
143
"MLA is enabled on a non-GPU platform; forcing chunked "
144
"prefill and prefix caching to be disabled.")