A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Bugfix] Fix FP8 torch._scaled_mm fallback for torch>2.5 with CUDA<12.4 (#10095)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Nov 6, 2024 at 19:54 UTC
4ab32566449558f2b5dbfbe44aeb6417e02e2e88
1 file changed
+2
-4
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+2
-4
@@ -7,8 +7,7 @@ from vllm.platforms import current_platform
7
8
# Input scaling factors are no longer optional in _scaled_mm starting
9
# from pytorch 2.5. Allocating a dummy tensor to pass as input_scale
10
-TORCH_DEVICE_IDENTITY = torch.ones(1).cuda() \
11
- if current_platform.is_rocm() else None
10
+TORCH_DEVICE_IDENTITY = torch.ones(1, dtype=torch.float32)
11
12
13
def cutlass_fp8_supported() -> bool:
@@ -166,8 +165,7 @@ def apply_fp8_linear(
165
166
# Making sure the dummy tensor is on the same device as the weight
167
global TORCH_DEVICE_IDENTITY
169
- if (TORCH_DEVICE_IDENTITY is not None
170
- and TORCH_DEVICE_IDENTITY.device != weight.device):
168
+ if TORCH_DEVICE_IDENTITY.device != weight.device:
169
TORCH_DEVICE_IDENTITY = TORCH_DEVICE_IDENTITY.to(weight.device)
170
171
# GEMM