@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5a9da2e6e9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix][Build/CI] Fix sparse CUTLASS compilation on CUDA [12.0, 12.2) (#11311)

Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>

Tyler Michael Smith committed Dec 18, 2024 at 21:43 UTC 5a9da2e6e952160a80936b0119364e789661c7a1
12 files changed +89 -20
CMakeLists.txt
+30 -9
@@ -273,15 +273,11 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
273 " in CUDA target architectures")
274 endif()
275
276 - #
277 - # The cutlass_scaled_mm cutlass_scaled_sparse_mm, and cutlass_compressor kernels
278 - # For Hopper (c3x, i.e. CUTLASS 3.x) require
276 + # The cutlass_scaled_mm kernels for Hopper (c3x, i.e. CUTLASS 3.x) require
277 # CUDA 12.0 or later (and only work on Hopper, 9.0/9.0a for now).
278 cuda_archs_loose_intersection(SCALED_MM_3X_ARCHS "9.0;9.0a" "${CUDA_ARCHS}")
279 if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
282 - set(SRCS "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu"
283 - "csrc/sparse/cutlass/sparse_compressor_c3x.cu"
284 - "csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu")
280 + set(SRCS "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu")
281 set_gencode_flags_for_srcs(
282 SRCS "${SRCS}"
283 CUDA_ARCHS "${SCALED_MM_3X_ARCHS}")
@@ -290,12 +286,12 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
286 message(STATUS "Building scaled_mm_c3x for archs: ${SCALED_MM_3X_ARCHS}")
287 else()
288 if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
293 - message(STATUS "Not building cutlass_c3x kernels as CUDA Compiler version is "
289 + message(STATUS "Not building scaled_mm_c3x as CUDA Compiler version is "
290 "not >= 12.0, we recommend upgrading to CUDA 12.0 or "
295 - "later if you intend on running FP8 sparse or quantized models on "
291 + "later if you intend on running FP8 quantized models on "
292 "Hopper.")
293 else()
298 - message(STATUS "Not building cutlass_c3x as no compatible archs found "
294 + message(STATUS "Not building scaled_mm_c3x as no compatible archs found "
295 "in CUDA target architectures")
296 endif()
297
@@ -329,6 +325,31 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
325 endif()
326 endif()
327
328 + #
329 + # 2:4 Sparse Kernels
330 +
331 + # The 2:4 sparse kernels cutlass_scaled_sparse_mm and cutlass_compressor
332 + # require CUDA 12.2 or later (and only work on Hopper, 9.0/9.0a for now).
333 + if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.2 AND SCALED_MM_3X_ARCHS)
334 + set(SRCS "csrc/sparse/cutlass/sparse_compressor_c3x.cu"
335 + "csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu")
336 + set_gencode_flags_for_srcs(
337 + SRCS "${SRCS}"
338 + CUDA_ARCHS "${SCALED_MM_3X_ARCHS}")
339 + list(APPEND VLLM_EXT_SRC "${SRCS}")
340 + list(APPEND VLLM_GPU_FLAGS "-DENABLE_SPARSE_SCALED_MM_C3X=1")
341 + message(STATUS "Building sparse_scaled_mm_c3x for archs: ${SCALED_MM_3X_ARCHS}")
342 + else()
343 + if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.2 AND SCALED_MM_3X_ARCHS)
344 + message(STATUS "Not building sparse_scaled_mm_c3x kernels as CUDA Compiler version is "
345 + "not >= 12.2, we recommend upgrading to CUDA 12.2 or later "
346 + "if you intend on running FP8 sparse quantized models on Hopper.")
347 + else()
348 + message(STATUS "Not building sparse_scaled_mm_c3x as no compatible archs found "
349 + "in CUDA target architectures")
350 + endif()
351 + endif()
352 +
353
354 #
355 # Machete kernels
csrc/ops.h
+2
@@ -163,6 +163,8 @@ void cutlass_scaled_mm_azp(torch::Tensor& out, torch::Tensor const& a,
163 c10::optional<torch::Tensor> const& azp,
164 c10::optional<torch::Tensor> const& bias);
165
166 +bool cutlass_sparse_scaled_mm_supported(int64_t cuda_device_capability);
167 +
168 void cutlass_scaled_sparse_mm(torch::Tensor& out, torch::Tensor const& a,
169 torch::Tensor const& b, torch::Tensor const& e,
170 torch::Tensor const& a_scales,
csrc/sparse/cutlass/sparse_compressor_c3x.cu
+3 -1
@@ -2,6 +2,7 @@
2 // clang-format off
3 #include <cudaTypedefs.h>
4
5 +#if defined CUDA_VERSION && CUDA_VERSION >= 12020
6 #include "sparse_scaled_mm_c3x.cuh"
7
8 #include "cutlass/numeric_conversion.h"
@@ -160,4 +161,5 @@ bool cutlass_sparse_compress_sm90(torch::Tensor& a_nzs, torch::Tensor& a_meta,
161 return cutlass_sparse_compress<int8_t, int32_t>(a_nzs, a_meta, a);
162 }
163 return false;
163 -}
\ No newline at end of file
164 +}
165 +#endif
csrc/sparse/cutlass/sparse_compressor_entry.cu
+2 -2
@@ -5,7 +5,7 @@
5
6 #include "cutlass_extensions/common.hpp"
7
8 -#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
8 +#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
9 bool cutlass_sparse_compress_sm90(torch::Tensor& a_nzs, torch::Tensor& a_meta,
10 torch::Tensor const& a);
11 #endif
@@ -28,7 +28,7 @@ bool cutlass_sparse_compress_entry(torch::Tensor& a_nzs, torch::Tensor& a_meta,
28 int32_t version_num = get_sm_version_num();
29
30 // Guard against compilation issues for sm90 kernels
31 -#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
31 +#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
32 if (version_num >= 90) {
33 return cutlass_sparse_compress_sm90(a_nzs, a_meta, a);
34 }
csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu
+1 -1
@@ -2,7 +2,7 @@
2 // clang-format off
3 #include <cudaTypedefs.h>
4
5 -#if defined CUDA_VERSION && CUDA_VERSION >= 12000
5 +#if defined CUDA_VERSION && CUDA_VERSION >= 12020
6 #include "sparse_scaled_mm_c3x.cuh"
7 // clang-format on
8
csrc/sparse/cutlass/sparse_scaled_mm_entry.cu
+13 -2
@@ -5,7 +5,18 @@
5
6 #include "cutlass_extensions/common.hpp"
7
8 -#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
8 +bool cutlass_sparse_scaled_mm_supported(int64_t cuda_device_capability) {
9 + // sparse CUTLASS kernels need at least
10 + // CUDA 12.2 and SM90 (Hopper)
11 +
12 +#if defined CUDA_VERSION
13 + return CUDA_VERSION >= 12020 && cuda_device_capability >= 90;
14 +#endif
15 +
16 + return false;
17 +}
18 +
19 +#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
20 void cutlass_scaled_sparse_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
21 torch::Tensor const& b,
22 torch::Tensor const& e,
@@ -43,7 +54,7 @@ void cutlass_scaled_sparse_mm(torch::Tensor& c, torch::Tensor const& a,
54 int32_t version_num = get_sm_version_num();
55
56 // Guard against compilation issues for sm90 kernels
46 -#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
57 +#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
58 if (version_num >= 90) {
59 cutlass_scaled_sparse_mm_sm90(c, a, bt_nzs, bt_meta, a_scales, b_scales,
60 bias);
csrc/torch_bindings.cpp
+7
@@ -321,6 +321,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
321 ops.def("cutlass_scaled_mm_supports_fp8(int cuda_device_capability) -> bool");
322 ops.impl("cutlass_scaled_mm_supports_fp8", &cutlass_scaled_mm_supports_fp8);
323
324 + // Check if cutlass sparse scaled_mm is supported for CUDA devices of the
325 + // given capability
326 + ops.def(
327 + "cutlass_sparse_scaled_mm_supported(int cuda_device_capability) -> bool");
328 + ops.impl("cutlass_sparse_scaled_mm_supported",
329 + &cutlass_sparse_scaled_mm_supported);
330 +
331 // CUTLASS sparse GEMM, supporting symmetric per-tensor or per-row/column
332 // quantization, as well as bias
333 ops.def(
tests/kernels/test_semi_structured.py
+4 -1
@@ -8,6 +8,8 @@ import pytest
8 import torch
9
10 from vllm import _custom_ops as ops
11 +from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
12 + sparse_cutlass_supported)
13 from vllm.platforms import current_platform
14
15 CUDA_DEVICES = [
@@ -102,10 +104,11 @@ def baseline_scaled_mm(a: torch.Tensor,
104 return output
105
106
105 -@pytest.mark.skipif(not current_platform.has_device_capability(90),
107 +@pytest.mark.skipif(not sparse_cutlass_supported(),
108 reason="Sparse FP8 is not yet supported on this GPU type.")
109 # Test working with a subset of A and B for sparse matmul
110 def test_cutlass_sparse_subset():
111 +
112 big_m = 1024
113 m, n, k = 512, 512, 512
114
tests/quantization/test_compressed_tensors.py
+5 -3
@@ -14,6 +14,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
14 CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8Fp8,
15 CompressedTensorsW8A8Int8, CompressedTensorsW8A16Fp8,
16 CompressedTensorsWNA16)
17 +from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
18 + sparse_cutlass_supported)
19 from vllm.platforms import current_platform
20
21
@@ -212,7 +214,7 @@ def test_compressed_tensors_kv_cache(vllm_runner):
214 assert output
215
216
215 -@pytest.mark.skipif(not current_platform.has_device_capability(90),
217 +@pytest.mark.skipif(not sparse_cutlass_supported(),
218 reason="Sparse FP8 is not yet supported on this GPU type.")
219 def _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy):
220 assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
@@ -254,7 +256,7 @@ def test_compressed_tensors_2of4_quant_fp8(vllm_runner, args_2of4):
256 assert output
257
258
257 -@pytest.mark.skipif(not current_platform.has_device_capability(90),
259 +@pytest.mark.skipif(not sparse_cutlass_supported(),
260 reason="Sparse FP8 is not yet supported on this GPU type.")
261 @pytest.mark.parametrize("args_2of4", [
262 ("nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Channel-Weight-testing",
@@ -279,7 +281,7 @@ def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):
281 assert output
282
283
282 -@pytest.mark.skipif(not current_platform.has_device_capability(90),
284 +@pytest.mark.skipif(not sparse_cutlass_supported(),
285 reason="Sparse FP8 is not yet supported on this GPU type.")
286 @pytest.mark.parametrize(
287 "args_2of4",
vllm/_custom_ops.py
+5
@@ -552,6 +552,11 @@ def cutlass_scaled_mm_azp(a: torch.Tensor,
552 return out
553
554
555 +def cutlass_sparse_scaled_mm_supported(cuda_device_capability: int) -> bool:
556 + return torch.ops._C.cutlass_sparse_scaled_mm_supported(
557 + cuda_device_capability)
558 +
559 +
560 def cutlass_sparse_compress(a: torch.Tensor) \
561 -> Tuple[torch.Tensor, torch.Tensor]:
562 """
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_24.py
+6 -1
@@ -9,7 +9,7 @@ from vllm import _custom_ops as ops
9 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10 CompressedTensorsScheme)
11 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
12 - convert_to_channelwise)
12 + convert_to_channelwise, sparse_cutlass_supported)
13 from vllm.model_executor.parameter import (BasevLLMParameter,
14 ChannelQuantScaleParameter,
15 ModelWeightParameter,
@@ -40,6 +40,11 @@ class CompressedTensors24(CompressedTensorsScheme):
40 params_dtype: torch.dtype, weight_loader: Callable,
41 **kwargs):
42
43 + if not sparse_cutlass_supported():
44 + raise ValueError(
45 + "Sparse CUTLASS not supported. vLLM must be built with"
46 + "CUDA 12.2 or later to use this feature")
47 +
48 self.output_dtype = params_dtype
49 layer.logical_widths = output_partition_sizes
50 self.weights_dtype: torch.dtype = self._get_params_dtype(params_dtype)
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+11
@@ -10,6 +10,17 @@ from vllm.platforms import current_platform
10 TORCH_DEVICE_IDENTITY = torch.ones(1, dtype=torch.float32)
11
12
13 +def sparse_cutlass_supported() -> bool:
14 + # sparse cutlass is not supported on Rocm
15 + if current_platform.is_rocm():
16 + return False
17 +
18 + capability_tuple = current_platform.get_device_capability()
19 + capability = -1 if capability_tuple is None else capability_tuple.to_int()
20 +
21 + return ops.cutlass_sparse_scaled_mm_supported(capability)
22 +
23 +
24 def cutlass_fp8_supported() -> bool:
25 # cutlass is not supported on Rocm
26 if current_platform.is_rocm():