A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix][Build/CI] Fix sparse CUTLASS compilation on CUDA [12.0, 12.2) (#11311)
Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>
Tyler Michael Smith committed
Dec 18, 2024 at 21:43 UTC
5a9da2e6e952160a80936b0119364e789661c7a1
12 files changed
+89
-20
CMakeLists.txt
+30
-9
@@ -273,15 +273,11 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
273
" in CUDA target architectures")
274
endif()
275
276
- #
277
- # The cutlass_scaled_mm cutlass_scaled_sparse_mm, and cutlass_compressor kernels
278
- # For Hopper (c3x, i.e. CUTLASS 3.x) require
276
+ # The cutlass_scaled_mm kernels for Hopper (c3x, i.e. CUTLASS 3.x) require
277
# CUDA 12.0 or later (and only work on Hopper, 9.0/9.0a for now).
278
cuda_archs_loose_intersection(SCALED_MM_3X_ARCHS "9.0;9.0a" "${CUDA_ARCHS}")
279
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
282
- set(SRCS "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu"
283
- "csrc/sparse/cutlass/sparse_compressor_c3x.cu"
284
- "csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu")
280
+ set(SRCS "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu")
281
set_gencode_flags_for_srcs(
282
SRCS "${SRCS}"
283
CUDA_ARCHS "${SCALED_MM_3X_ARCHS}")
@@ -290,12 +286,12 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
286
message(STATUS "Building scaled_mm_c3x for archs: ${SCALED_MM_3X_ARCHS}")
287
else()
288
if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
293
- message(STATUS "Not building cutlass_c3x kernels as CUDA Compiler version is "
289
+ message(STATUS "Not building scaled_mm_c3x as CUDA Compiler version is "
290
"not >= 12.0, we recommend upgrading to CUDA 12.0 or "
295
- "later if you intend on running FP8 sparse or quantized models on "
291
+ "later if you intend on running FP8 quantized models on "
292
"Hopper.")
293
else()
298
- message(STATUS "Not building cutlass_c3x as no compatible archs found "
294
+ message(STATUS "Not building scaled_mm_c3x as no compatible archs found "
295
"in CUDA target architectures")
296
endif()
297
@@ -329,6 +325,31 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
325
endif()
326
endif()
327
328
+ #
329
+ # 2:4 Sparse Kernels
330
+
331
+ # The 2:4 sparse kernels cutlass_scaled_sparse_mm and cutlass_compressor
332
+ # require CUDA 12.2 or later (and only work on Hopper, 9.0/9.0a for now).
333
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.2 AND SCALED_MM_3X_ARCHS)
334
+ set(SRCS "csrc/sparse/cutlass/sparse_compressor_c3x.cu"
335
+ "csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu")
336
+ set_gencode_flags_for_srcs(
337
+ SRCS "${SRCS}"
338
+ CUDA_ARCHS "${SCALED_MM_3X_ARCHS}")
339
+ list(APPEND VLLM_EXT_SRC "${SRCS}")
340
+ list(APPEND VLLM_GPU_FLAGS "-DENABLE_SPARSE_SCALED_MM_C3X=1")
341
+ message(STATUS "Building sparse_scaled_mm_c3x for archs: ${SCALED_MM_3X_ARCHS}")
342
+ else()
343
+ if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.2 AND SCALED_MM_3X_ARCHS)
344
+ message(STATUS "Not building sparse_scaled_mm_c3x kernels as CUDA Compiler version is "
345
+ "not >= 12.2, we recommend upgrading to CUDA 12.2 or later "
346
+ "if you intend on running FP8 sparse quantized models on Hopper.")
347
+ else()
348
+ message(STATUS "Not building sparse_scaled_mm_c3x as no compatible archs found "
349
+ "in CUDA target architectures")
350
+ endif()
351
+ endif()
352
+
353
354
#
355
# Machete kernels
csrc/ops.h
+2
@@ -163,6 +163,8 @@ void cutlass_scaled_mm_azp(torch::Tensor& out, torch::Tensor const& a,
163
c10::optional<torch::Tensor> const& azp,
164
c10::optional<torch::Tensor> const& bias);
165
166
+bool cutlass_sparse_scaled_mm_supported(int64_t cuda_device_capability);
167
+
168
void cutlass_scaled_sparse_mm(torch::Tensor& out, torch::Tensor const& a,
169
torch::Tensor const& b, torch::Tensor const& e,
170
torch::Tensor const& a_scales,
csrc/sparse/cutlass/sparse_compressor_c3x.cu
+3
-1
@@ -2,6 +2,7 @@
2
// clang-format off
3
#include <cudaTypedefs.h>
4
5
+#if defined CUDA_VERSION && CUDA_VERSION >= 12020
6
#include "sparse_scaled_mm_c3x.cuh"
7
8
#include "cutlass/numeric_conversion.h"
@@ -160,4 +161,5 @@ bool cutlass_sparse_compress_sm90(torch::Tensor& a_nzs, torch::Tensor& a_meta,
161
return cutlass_sparse_compress<int8_t, int32_t>(a_nzs, a_meta, a);
162
}
163
return false;
163
-}
\ No newline at end of file
164
+}
165
+#endif
csrc/sparse/cutlass/sparse_compressor_entry.cu
+2
-2
@@ -5,7 +5,7 @@
5
6
#include "cutlass_extensions/common.hpp"
7
8
-#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
8
+#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
9
bool cutlass_sparse_compress_sm90(torch::Tensor& a_nzs, torch::Tensor& a_meta,
10
torch::Tensor const& a);
11
#endif
@@ -28,7 +28,7 @@ bool cutlass_sparse_compress_entry(torch::Tensor& a_nzs, torch::Tensor& a_meta,
28
int32_t version_num = get_sm_version_num();
29
30
// Guard against compilation issues for sm90 kernels
31
-#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
31
+#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
32
if (version_num >= 90) {
33
return cutlass_sparse_compress_sm90(a_nzs, a_meta, a);
34
}
csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu
+1
-1
@@ -2,7 +2,7 @@
2
// clang-format off
3
#include <cudaTypedefs.h>
4
5
-#if defined CUDA_VERSION && CUDA_VERSION >= 12000
5
+#if defined CUDA_VERSION && CUDA_VERSION >= 12020
6
#include "sparse_scaled_mm_c3x.cuh"
7
// clang-format on
8
csrc/sparse/cutlass/sparse_scaled_mm_entry.cu
+13
-2
@@ -5,7 +5,18 @@
5
6
#include "cutlass_extensions/common.hpp"
7
8
-#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
8
+bool cutlass_sparse_scaled_mm_supported(int64_t cuda_device_capability) {
9
+ // sparse CUTLASS kernels need at least
10
+ // CUDA 12.2 and SM90 (Hopper)
11
+
12
+#if defined CUDA_VERSION
13
+ return CUDA_VERSION >= 12020 && cuda_device_capability >= 90;
14
+#endif
15
+
16
+ return false;
17
+}
18
+
19
+#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
20
void cutlass_scaled_sparse_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
21
torch::Tensor const& b,
22
torch::Tensor const& e,
@@ -43,7 +54,7 @@ void cutlass_scaled_sparse_mm(torch::Tensor& c, torch::Tensor const& a,
54
int32_t version_num = get_sm_version_num();
55
56
// Guard against compilation issues for sm90 kernels
46
-#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
57
+#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
58
if (version_num >= 90) {
59
cutlass_scaled_sparse_mm_sm90(c, a, bt_nzs, bt_meta, a_scales, b_scales,
60
bias);
csrc/torch_bindings.cpp
+7
@@ -321,6 +321,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
321
ops.def("cutlass_scaled_mm_supports_fp8(int cuda_device_capability) -> bool");
322
ops.impl("cutlass_scaled_mm_supports_fp8", &cutlass_scaled_mm_supports_fp8);
323
324
+ // Check if cutlass sparse scaled_mm is supported for CUDA devices of the
325
+ // given capability
326
+ ops.def(
327
+ "cutlass_sparse_scaled_mm_supported(int cuda_device_capability) -> bool");
328
+ ops.impl("cutlass_sparse_scaled_mm_supported",
329
+ &cutlass_sparse_scaled_mm_supported);
330
+
331
// CUTLASS sparse GEMM, supporting symmetric per-tensor or per-row/column
332
// quantization, as well as bias
333
ops.def(
tests/kernels/test_semi_structured.py
+4
-1
@@ -8,6 +8,8 @@ import pytest
8
import torch
9
10
from vllm import _custom_ops as ops
11
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
12
+ sparse_cutlass_supported)
13
from vllm.platforms import current_platform
14
15
CUDA_DEVICES = [
@@ -102,10 +104,11 @@ def baseline_scaled_mm(a: torch.Tensor,
104
return output
105
106
105
-@pytest.mark.skipif(not current_platform.has_device_capability(90),
107
+@pytest.mark.skipif(not sparse_cutlass_supported(),
108
reason="Sparse FP8 is not yet supported on this GPU type.")
109
# Test working with a subset of A and B for sparse matmul
110
def test_cutlass_sparse_subset():
111
+
112
big_m = 1024
113
m, n, k = 512, 512, 512
114
tests/quantization/test_compressed_tensors.py
+5
-3
@@ -14,6 +14,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
14
CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8Fp8,
15
CompressedTensorsW8A8Int8, CompressedTensorsW8A16Fp8,
16
CompressedTensorsWNA16)
17
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
18
+ sparse_cutlass_supported)
19
from vllm.platforms import current_platform
20
21
@@ -212,7 +214,7 @@ def test_compressed_tensors_kv_cache(vllm_runner):
214
assert output
215
216
215
-@pytest.mark.skipif(not current_platform.has_device_capability(90),
217
+@pytest.mark.skipif(not sparse_cutlass_supported(),
218
reason="Sparse FP8 is not yet supported on this GPU type.")
219
def _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy):
220
assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
@@ -254,7 +256,7 @@ def test_compressed_tensors_2of4_quant_fp8(vllm_runner, args_2of4):
256
assert output
257
258
257
-@pytest.mark.skipif(not current_platform.has_device_capability(90),
259
+@pytest.mark.skipif(not sparse_cutlass_supported(),
260
reason="Sparse FP8 is not yet supported on this GPU type.")
261
@pytest.mark.parametrize("args_2of4", [
262
("nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Channel-Weight-testing",
@@ -279,7 +281,7 @@ def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):
281
assert output
282
283
282
-@pytest.mark.skipif(not current_platform.has_device_capability(90),
284
+@pytest.mark.skipif(not sparse_cutlass_supported(),
285
reason="Sparse FP8 is not yet supported on this GPU type.")
286
@pytest.mark.parametrize(
287
"args_2of4",
vllm/_custom_ops.py
+5
@@ -552,6 +552,11 @@ def cutlass_scaled_mm_azp(a: torch.Tensor,
552
return out
553
554
555
+def cutlass_sparse_scaled_mm_supported(cuda_device_capability: int) -> bool:
556
+ return torch.ops._C.cutlass_sparse_scaled_mm_supported(
557
+ cuda_device_capability)
558
+
559
+
560
def cutlass_sparse_compress(a: torch.Tensor) \
561
-> Tuple[torch.Tensor, torch.Tensor]:
562
"""
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_24.py
+6
-1
@@ -9,7 +9,7 @@ from vllm import _custom_ops as ops
9
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10
CompressedTensorsScheme)
11
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
12
- convert_to_channelwise)
12
+ convert_to_channelwise, sparse_cutlass_supported)
13
from vllm.model_executor.parameter import (BasevLLMParameter,
14
ChannelQuantScaleParameter,
15
ModelWeightParameter,
@@ -40,6 +40,11 @@ class CompressedTensors24(CompressedTensorsScheme):
40
params_dtype: torch.dtype, weight_loader: Callable,
41
**kwargs):
42
43
+ if not sparse_cutlass_supported():
44
+ raise ValueError(
45
+ "Sparse CUTLASS not supported. vLLM must be built with"
46
+ "CUDA 12.2 or later to use this feature")
47
+
48
self.output_dtype = params_dtype
49
layer.logical_widths = output_partition_sizes
50
self.weights_dtype: torch.dtype = self._get_params_dtype(params_dtype)
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+11
@@ -10,6 +10,17 @@ from vllm.platforms import current_platform
10
TORCH_DEVICE_IDENTITY = torch.ones(1, dtype=torch.float32)
11
12
13
+def sparse_cutlass_supported() -> bool:
14
+ # sparse cutlass is not supported on Rocm
15
+ if current_platform.is_rocm():
16
+ return False
17
+
18
+ capability_tuple = current_platform.get_device_capability()
19
+ capability = -1 if capability_tuple is None else capability_tuple.to_int()
20
+
21
+ return ops.cutlass_sparse_scaled_mm_supported(capability)
22
+
23
+
24
def cutlass_fp8_supported() -> bool:
25
# cutlass is not supported on Rocm
26
if current_platform.is_rocm():