A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 1, 23:45 UTC
[Bugfix][Compile] Guard per_token_group_fp8_quant lookup on non-CUDA platforms (#44476)
QiliangCui2023 committed
Jun 4, 2026 at 06:31 UTC
9354fb1ba5ca652835d0e6c88b15c3a925e1aab0
2 files changed
+7
-4
vllm/compilation/passes/fusion/matcher_utils.py
+4
-2
@@ -36,10 +36,12 @@ QUANT_OPS: dict[QuantKey, OpOverload] = {
36
kFp8StaticTensorSym: torch.ops._C.static_scaled_fp8_quant.default, # noqa: E501
37
kFp8DynamicTensorSym: torch.ops._C.dynamic_scaled_fp8_quant.default, # noqa: E501
38
kFp8DynamicTokenSym: torch.ops._C.dynamic_per_token_scaled_fp8_quant.default, # noqa: E501
39
- kFp8Dynamic128Sym: torch.ops._C.per_token_group_fp8_quant.default, # noqa: E501
40
- kFp8Dynamic64Sym: torch.ops._C.per_token_group_fp8_quant.default, # noqa: E501
39
}
40
41
+if hasattr(torch.ops._C, "per_token_group_fp8_quant"):
42
+ QUANT_OPS[kFp8Dynamic128Sym] = torch.ops._C.per_token_group_fp8_quant.default # noqa: E501
43
+ QUANT_OPS[kFp8Dynamic64Sym] = torch.ops._C.per_token_group_fp8_quant.default # noqa: E501
44
+
45
if current_platform.is_cuda() and hasattr(torch.ops._C, "scaled_fp4_quant"):
46
QUANT_OPS[kNvfp4Dynamic] = torch.ops._C.scaled_fp4_quant.out # noqa: E501
47
vllm/compilation/passes/fusion/rms_quant_fusion.py
+3
-2
@@ -84,9 +84,10 @@ QUANT_OPS: dict[QuantKey, OpOverload] = {
84
kFp8StaticTensorSym: torch.ops._C.static_scaled_fp8_quant.default, # noqa: E501
85
kFp8DynamicTensorSym: torch.ops._C.dynamic_scaled_fp8_quant.default, # noqa: E501
86
kFp8DynamicTokenSym: torch.ops._C.dynamic_per_token_scaled_fp8_quant.default, # noqa: E501
87
- kFp8Dynamic128Sym: torch.ops._C.per_token_group_fp8_quant.default, # noqa: E501
88
- kFp8Dynamic64Sym: torch.ops._C.per_token_group_fp8_quant.default, # noqa: E501
87
}
88
+if hasattr(torch.ops._C, "per_token_group_fp8_quant"):
89
+ QUANT_OPS[kFp8Dynamic128Sym] = torch.ops._C.per_token_group_fp8_quant.default # noqa: E501
90
+ QUANT_OPS[kFp8Dynamic64Sym] = torch.ops._C.per_token_group_fp8_quant.default # noqa: E501
91
if current_platform.is_cuda() and hasattr(torch.ops._C, "scaled_fp4_quant"):
92
QUANT_OPS[kNvfp4Dynamic] = torch.ops._C.scaled_fp4_quant.out
93