@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 276738ce0f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Fix broken CPU compressed-tensors test (#11338)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Dec 20, 2024 at 01:37 UTC 276738ce0f6aac48ace36bc79aa4a0765fccdfb2
1 file changed +2 -4
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+2 -4
@@ -11,8 +11,7 @@ TORCH_DEVICE_IDENTITY = torch.ones(1, dtype=torch.float32)
11
12
13 def sparse_cutlass_supported() -> bool:
14 - # sparse cutlass is not supported on Rocm
15 - if current_platform.is_rocm():
14 + if not current_platform.is_cuda():
15 return False
16
17 capability_tuple = current_platform.get_device_capability()
@@ -22,8 +21,7 @@ def sparse_cutlass_supported() -> bool:
21
22
23 def cutlass_fp8_supported() -> bool:
25 - # cutlass is not supported on Rocm
26 - if current_platform.is_rocm():
24 + if not current_platform.is_cuda():
25 return False
26
27 capability_tuple = current_platform.get_device_capability()