A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
fix incompatibililty with non cuda platform for nvfp4 (#23478)
Signed-off-by: Lu Fang <fanglu@fb.com> Co-authored-by: Lucia (Lu) Fang <fanglu@meta.com>
Lucia Fang committed
Aug 24, 2025 at 15:35 UTC
c7fc6b1354a20f5dbdd2fb806cd4b7da27d46f63
1 file changed
+3
-1
vllm/compilation/fusion.py
+3
-1
@@ -47,8 +47,10 @@ QUANT_OPS: dict[QuantKey, OpOverload] = {
47
torch.ops._C.dynamic_scaled_fp8_quant.default, # noqa: E501
48
kFp8DynamicTokenSym:
49
torch.ops._C.dynamic_per_token_scaled_fp8_quant.default, # noqa: E501
50
- kNvfp4Quant: torch.ops._C.scaled_fp4_quant.default, # noqa: E501
50
}
51
+if current_platform.is_cuda() and hasattr(torch.ops._C, "scaled_fp4_quant"):
52
+ QUANT_OPS[
53
+ kNvfp4Quant] = torch.ops._C.scaled_fp4_quant.default # noqa: E501
54
55
56
class FusedRMSQuantKey(NamedTuple):