A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 05:45 UTC
[Bug] Fix fp8 deepgemm batch invariant (#37718)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Mar 22, 2026 at 08:57 UTC
77d24c4bfedc9812a703b865cf5935f4e941660f
1 file changed
+5
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+5
@@ -305,6 +305,11 @@ def _flashinfer_fp8_blockscale_gemm_impl(
305
)
306
return output
307
308
+ from vllm.model_executor.layers.batch_invariant import vllm_is_batch_invariant
309
+
310
+ if vllm_is_batch_invariant():
311
+ return run_deepgemm(input, weight, weight_scale)
312
+
313
condition = input.shape[0] < 32
314
315
# PyTorch's torch.compile cannot handle input-dependent control flow in standard