A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 06:00 UTC
[BugFix] add max-num-batched-token to scheduler hash (#29829)
Signed-off-by: Boyuan Feng <boyuan@meta.com>
Boyuan Feng committed
Dec 2, 2025 at 00:55 UTC
70fb77b4dcc3cfb368831d1aba8e1e2dca7c31a9
2 files changed
+12
-6
vllm/config/scheduler.py
+12
-2
@@ -175,9 +175,19 @@ class SchedulerConfig:
175
excluding anything before input ids/embeddings and after
176
the final hidden states.
177
"""
178
- # no factors to consider.
179
- # this config will not affect the computation graph.
178
factors: list[Any] = []
179
+
180
+ # max_num_batched_tokens need to be included in the hash due
181
+ # to two reasons:
182
+ # 1. LoRA creates static buffers based on max_num_batched_tokens.
183
+ # The tensor sizes and strides get captured in the torch.compile
184
+ # graph explicitly.
185
+ # 2. Inductor decides whether using 32-bit or 64-bit indexing integer
186
+ # based on the data sizes. `max_num_batched_tokens` has an
187
+ # impact on that. For more details, please check
188
+ # https://github.com/vllm-project/vllm/issues/29585
189
+ factors.append(self.max_num_batched_tokens)
190
+
191
hash_str = safe_hash(str(factors).encode(), usedforsecurity=False).hexdigest()
192
return hash_str
193
vllm/config/vllm.py
-4
@@ -265,10 +265,6 @@ class VllmConfig:
265
vllm_factors.append("None")
266
if self.lora_config:
267
vllm_factors.append(self.lora_config.compute_hash())
268
- # LoRA creates static buffers based on max_num_batched_tokens.
269
- # The tensor sizes and strides get captured in the torch.compile
270
- # graph explicitly.
271
- vllm_factors.append(str(self.scheduler_config.max_num_batched_tokens))
268
else:
269
vllm_factors.append("None")
270
if self.speculative_config: