A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 21:30 UTC
[Perf] Enable FlashInfer DeepGEMM swapAB on SM90 by default (#34924)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Michael Goin committed
Feb 23, 2026 at 20:34 UTC
a4bd661fb33b94e952b598624f1b9e60f380d325
2 files changed
+7
-2
tests/compile/fusions_e2e/test_tp1_quant.py
+5
@@ -5,6 +5,7 @@ from collections.abc import Callable
5
import pytest
6
7
from vllm.config import PassConfig
8
+from vllm.utils.flashinfer import is_flashinfer_fp8_blockscale_gemm_supported
9
10
from .common import (
11
INDUCTOR_GRAPH_PARTITION,
@@ -50,6 +51,10 @@ def test_tp1_fp8_fusions(
51
run_e2e_fusion_test,
52
monkeypatch,
53
):
54
+ if use_deepgemm and is_flashinfer_fp8_blockscale_gemm_supported():
55
+ # Flashinfer block FP8 GEMM has internal quantization, so it can't
56
+ # be fused with other ops.
57
+ pytest.skip("FlashInfer block FP8 GEMM not supported")
58
if use_deepgemm and is_blackwell():
59
# TODO(luka) DeepGEMM uses different quants, matching not supported
60
# - on Blackwell, uses a special quant fp8, currently not supported
vllm/envs.py
+2
-2
@@ -159,7 +159,7 @@ if TYPE_CHECKING:
159
"relax",
160
] = "relax"
161
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
162
- VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = False
162
+ VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
163
VLLM_USE_FLASHINFER_MOE_FP16: bool = False
164
VLLM_USE_FLASHINFER_MOE_FP8: bool = False
165
VLLM_USE_FLASHINFER_MOE_FP4: bool = False
@@ -1198,7 +1198,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
1198
# Allow use of FlashInfer FP8 block-scale GEMM for linear layers.
1199
# This uses TensorRT-LLM kernels and requires SM90+ (Hopper).
1200
"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(
1201
- int(os.getenv("VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER", "0"))
1201
+ int(os.getenv("VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER", "1"))
1202
),
1203
# Allow use of FlashInfer BF16 MoE kernels for fused moe ops.
1204
"VLLM_USE_FLASHINFER_MOE_FP16": lambda: bool(