A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 00:30 UTC
[BugFix] Don’t compute reorder threshold when there are no attention groups (#27861)
Huamin Li committed
Oct 31, 2025 at 04:36 UTC
933cdea44061cb19a99421d2d2e51535e7f21216
1 file changed
+5
vllm/v1/worker/gpu_model_runner.py
+5
@@ -4149,6 +4149,11 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
4149
group.get_metadata_builder().reorder_batch_threshold
4150
for group in self._attn_group_iterator()
4151
]
4152
+ # If there are no attention groups (attention-free model) or no backend
4153
+ # reports a threshold, leave reordering disabled.
4154
+ if len(reorder_batch_thresholds) == 0:
4155
+ self.reorder_batch_threshold = None
4156
+ return
4157
self.reorder_batch_threshold = reduce(min_none_high, reorder_batch_thresholds)
4158
4159
def _find_compatible_block_sizes(