@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4ab3256644
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Bugfix] Fix FP8 torch._scaled_mm fallback for torch>2.5 with CUDA<12.4 (#10095)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Nov 6, 2024 at 19:54 UTC 4ab32566449558f2b5dbfbe44aeb6417e02e2e88
1 file changed +2 -4
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+2 -4
@@ -7,8 +7,7 @@ from vllm.platforms import current_platform
7
8 # Input scaling factors are no longer optional in _scaled_mm starting
9 # from pytorch 2.5. Allocating a dummy tensor to pass as input_scale
10 -TORCH_DEVICE_IDENTITY = torch.ones(1).cuda() \
11 - if current_platform.is_rocm() else None
10 +TORCH_DEVICE_IDENTITY = torch.ones(1, dtype=torch.float32)
11
12
13 def cutlass_fp8_supported() -> bool:
@@ -166,8 +165,7 @@ def apply_fp8_linear(
165
166 # Making sure the dummy tensor is on the same device as the weight
167 global TORCH_DEVICE_IDENTITY
169 - if (TORCH_DEVICE_IDENTITY is not None
170 - and TORCH_DEVICE_IDENTITY.device != weight.device):
168 + if TORCH_DEVICE_IDENTITY.device != weight.device:
169 TORCH_DEVICE_IDENTITY = TORCH_DEVICE_IDENTITY.to(weight.device)
170
171 # GEMM