A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Feature] Enable E8M0 by Default on Hopper for DeepGEMM, 5% E2E throughput improvement (#26197)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Wentao Ye committed
Oct 8, 2025 at 03:33 UTC
f8607863d88b7cb18c3b050b2c93f63532ac428a
3 files changed
+10
-29
vllm/envs.py
-7
@@ -142,7 +142,6 @@ if TYPE_CHECKING:
142
VLLM_TPU_USING_PATHWAYS: bool = False
143
VLLM_USE_DEEP_GEMM: bool = True
144
VLLM_USE_DEEP_GEMM_E8M0: bool = True
145
- VLLM_USE_DEEP_GEMM_E8M0_HOPPER: bool = False
145
VLLM_SKIP_DEEP_GEMM_WARMUP: bool = False
146
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
147
VLLM_USE_FLASHINFER_MOE_FP16: bool = False
@@ -1055,11 +1054,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
1054
"VLLM_USE_DEEP_GEMM_E8M0": lambda: bool(
1055
int(os.getenv("VLLM_USE_DEEP_GEMM_E8M0", "1"))
1056
),
1058
- # TODO(wentao): unify the two E8M0 flags after verifying the correctness.
1059
- # Whether to use E8M0 scaling when DeepGEMM is used on Hopper GPUs.
1060
- "VLLM_USE_DEEP_GEMM_E8M0_HOPPER": lambda: bool(
1061
- int(os.getenv("VLLM_USE_DEEP_GEMM_E8M0_HOPPER", "0"))
1062
- ),
1057
# DeepGemm JITs the kernels on-demand. The warmup attempts to make DeepGemm
1058
# JIT all the required kernels before model execution so there is no
1059
# JIT'ing in the hot-path. However, this warmup increases the engine
@@ -1433,7 +1427,6 @@ def compute_hash() -> str:
1427
"VLLM_DISABLED_KERNELS",
1428
"VLLM_USE_DEEP_GEMM",
1429
"VLLM_USE_DEEP_GEMM_E8M0",
1436
- "VLLM_USE_DEEP_GEMM_E8M0_HOPPER",
1430
"VLLM_USE_TRTLLM_FP4_GEMM",
1431
"VLLM_USE_FUSED_MOE_GROUPED_TOPK",
1432
"VLLM_USE_FLASHINFER_MOE_FP16",
vllm/transformers_utils/config.py
+7
-7
@@ -629,25 +629,25 @@ def get_config(
629
630
if quantization_config is not None:
631
config.quantization_config = quantization_config
632
- # auto-enable DeepGEMM UE8M0 on Hopper if model config requests it
632
+ # auto-enable DeepGEMM UE8M0 if model config requests it
633
scale_fmt = quantization_config.get("scale_fmt", None)
634
if scale_fmt in ("ue8m0",):
635
- if not envs.is_set("VLLM_USE_DEEP_GEMM_E8M0_HOPPER"):
636
- os.environ["VLLM_USE_DEEP_GEMM_E8M0_HOPPER"] = "1"
635
+ if not envs.is_set("VLLM_USE_DEEP_GEMM_E8M0"):
636
+ os.environ["VLLM_USE_DEEP_GEMM_E8M0"] = "1"
637
logger.info_once(
638
(
639
"Detected quantization_config.scale_fmt=%s; "
640
- "enabling Hopper UE8M0."
640
+ "enabling UE8M0 for DeepGEMM."
641
),
642
scale_fmt,
643
)
644
- elif not envs.VLLM_USE_DEEP_GEMM_E8M0_HOPPER:
644
+ elif not envs.VLLM_USE_DEEP_GEMM_E8M0:
645
logger.warning_once(
646
(
647
"Model config requests UE8M0 "
648
"(quantization_config.scale_fmt=%s), but "
649
- "VLLM_USE_DEEP_GEMM_E8M0_HOPPER=0 is set; "
650
- "Hopper UE8M0 disabled."
649
+ "VLLM_USE_DEEP_GEMM_E8M0=0 is set; "
650
+ "UE8M0 for DeepGEMM disabled."
651
),
652
scale_fmt,
653
)
vllm/utils/deep_gemm.py
+3
-15
@@ -29,12 +29,7 @@ def is_deep_gemm_supported() -> bool:
29
current_platform.is_device_capability(90)
30
or current_platform.is_device_capability(100)
31
)
32
- return (
33
- envs.VLLM_USE_DEEP_GEMM
34
- and has_deep_gemm()
35
- and is_supported_arch
36
- and not envs.VLLM_USE_FLASHINFER_MOE_FP8
37
- )
32
+ return envs.VLLM_USE_DEEP_GEMM and has_deep_gemm() and is_supported_arch
33
34
35
@functools.cache
@@ -58,15 +53,8 @@ def is_deep_gemm_e8m0_used() -> bool:
53
logger.info_once("DeepGEMM E8M0 disabled: FlashInfer MOE is enabled.")
54
return False
55
61
- if current_platform.is_device_capability(100) and envs.VLLM_USE_DEEP_GEMM_E8M0:
62
- logger.info_once("DeepGEMM E8M0 enabled on Blackwell GPU.")
63
- return True
64
-
65
- if (
66
- current_platform.is_device_capability(90)
67
- and envs.VLLM_USE_DEEP_GEMM_E8M0_HOPPER
68
- ):
69
- logger.info_once("DeepGEMM E8M0 enabled on Hopper GPU.")
56
+ if envs.VLLM_USE_DEEP_GEMM_E8M0:
57
+ logger.info_once("DeepGEMM E8M0 enabled on current platform.")
58
return True
59
60
logger.info_once("DeepGEMM E8M0 disabled on current configuration.")