A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] Fix broken CPU compressed-tensors test (#11338)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Dec 20, 2024 at 01:37 UTC
276738ce0f6aac48ace36bc79aa4a0765fccdfb2
1 file changed
+2
-4
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+2
-4
@@ -11,8 +11,7 @@ TORCH_DEVICE_IDENTITY = torch.ones(1, dtype=torch.float32)
11
12
13
def sparse_cutlass_supported() -> bool:
14
- # sparse cutlass is not supported on Rocm
15
- if current_platform.is_rocm():
14
+ if not current_platform.is_cuda():
15
return False
16
17
capability_tuple = current_platform.get_device_capability()
@@ -22,8 +21,7 @@ def sparse_cutlass_supported() -> bool:
21
22
23
def cutlass_fp8_supported() -> bool:
25
- # cutlass is not supported on Rocm
26
- if current_platform.is_rocm():
24
+ if not current_platform.is_cuda():
25
return False
26
27
capability_tuple = current_platform.get_device_capability()