A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Bugfix] Disable w16a16 2of4 sparse CompressedTensors24 (#12417)
Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com> Co-authored-by: mgoin <michael@neuralmagic.com>
Tyler Michael Smith committed
Jan 26, 2025 at 06:59 UTC
aa2cd2c43d1d19ece0f3b36ad716c3a9b8a2def0
6 files changed
+263
-169
tests/kernels/test_cutlass.py
+3
-25
@@ -2,7 +2,7 @@
2
3
Run `pytest tests/kernels/test_cutlass.py`.
4
"""
5
-from typing import Optional, Type
5
+from typing import Type
6
7
import pytest
8
import torch
@@ -11,6 +11,8 @@ from tests.kernels.utils import opcheck
11
from vllm import _custom_ops as ops
12
from vllm.platforms import current_platform
13
14
+from .utils import baseline_scaled_mm, to_fp8, to_int8
15
+
16
MNK_FACTORS = [
17
(1, 256, 128),
18
(1, 16384, 1024),
@@ -41,34 +43,10 @@ capability = current_platform.get_device_capability()
43
capability = capability[0] * 10 + capability[1]
44
45
44
-def to_fp8(tensor: torch.Tensor):
45
- finfo = torch.finfo(torch.float8_e4m3fn)
46
- return torch.round(tensor.clamp(
47
- min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
48
-
49
-
50
-def to_int8(tensor: torch.Tensor):
51
- return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
52
-
53
-
46
def rand_int8(shape: tuple, device: str = "cuda"):
47
return to_int8(torch.rand(shape, device=device) * 255 - 128)
48
49
58
-def baseline_scaled_mm(a: torch.Tensor,
59
- b: torch.Tensor,
60
- scale_a: torch.Tensor,
61
- scale_b: torch.Tensor,
62
- out_dtype: Type[torch.dtype],
63
- bias: Optional[torch.Tensor] = None) -> torch.Tensor:
64
- output = (scale_a * (scale_b * (torch.mm(
65
- a.to(dtype=torch.float32), b.to(dtype=torch.float32))))).to(out_dtype)
66
- if bias is not None:
67
- output = output + bias
68
-
69
- return output
70
-
71
-
50
def cutlass_fp8_gemm_helper(m: int,
51
n: int,
52
k: int,
tests/kernels/test_cutlass_2of4_sparse.py
new
+214
@@ -0,0 +1,214 @@
1
+"""Tests for sparse cutlass kernels
2
+
3
+Run `pytest tests/kernels/test_semi_structured.py`.
4
+"""
5
+from typing import Tuple, Type
6
+
7
+import pytest
8
+import torch
9
+import torch.nn.functional as F
10
+
11
+from vllm import _custom_ops as ops
12
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
13
+ sparse_cutlass_supported)
14
+from vllm.platforms import current_platform
15
+
16
+from .utils import baseline_scaled_mm, to_fp8, to_int8
17
+
18
+CUDA_DEVICES = [
19
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
20
+]
21
+
22
+capability = current_platform.get_device_capability()
23
+capability = capability[0] * 10 + capability[1]
24
+
25
+
26
+def to_bf16(tensor: torch.Tensor) -> torch.Tensor:
27
+ return tensor.to(dtype=torch.bfloat16)
28
+
29
+
30
+def to_fp16(tensor: torch.Tensor) -> torch.Tensor:
31
+ return tensor.to(dtype=torch.float16)
32
+
33
+
34
+def prune_to_2_4(tensor):
35
+ # Reshape tensor to [N, 4] where N is number of groups of 4
36
+ original_shape = tensor.shape
37
+ reshaped = tensor.reshape(-1, 4)
38
+
39
+ # Get indices of top 2 absolute values in each group of 4
40
+ _, indices = torch.topk(torch.abs(reshaped), k=2, dim=1)
41
+
42
+ # Create binary mask
43
+ mask = torch.zeros_like(reshaped)
44
+ mask.scatter_(dim=1,
45
+ index=indices,
46
+ src=torch.ones_like(indices, dtype=mask.dtype))
47
+
48
+ # Apply mask and reshape back
49
+ pruned = reshaped * mask
50
+
51
+ # Turn all -0.0 to 0.0
52
+ pruned[pruned == -0.0] = 0.0
53
+
54
+ return pruned.reshape(original_shape)
55
+
56
+
57
+def make_rand_sparse_tensors(
58
+ dtype: torch.dtype, m: int, n: int, k: int
59
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
60
+ a = torch.randn((m, k), device='cuda') * 5
61
+ b = torch.randn((n, k), device='cuda').t() * 5
62
+
63
+ b = prune_to_2_4(b.t()).t()
64
+
65
+ if dtype == torch.int8:
66
+ a, b = to_int8(a), to_int8(b)
67
+ elif dtype == torch.float8_e4m3fn:
68
+ a, b = to_fp8(a), to_fp8(b)
69
+ elif dtype == torch.float16:
70
+ a, b = to_fp16(a), to_fp16(b)
71
+ elif dtype == torch.bfloat16:
72
+ a, b = to_bf16(a), to_bf16(b)
73
+ else:
74
+ raise ValueError("unsupported dtype")
75
+
76
+ b_compressed, e = ops.cutlass_sparse_compress(b.t())
77
+
78
+ # Compressed B, Metadata, Original A, B
79
+ return b_compressed, e, a, b
80
+
81
+
82
+@pytest.mark.skipif(not sparse_cutlass_supported(),
83
+ reason="Sparse CUTLASS is not supported on this GPU type.")
84
+# Test working with a subset of A and B for sparse matmul
85
+def test_cutlass_sparse_subset():
86
+
87
+ big_m = 1024
88
+ m, n, k = 512, 512, 512
89
+
90
+ # Create tensors
91
+ b_comp, e, whole_a, b = make_rand_sparse_tensors(torch.float8_e4m3fn,
92
+ big_m, n, k)
93
+ a = whole_a[0:m, 0:k]
94
+ scale_a = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
95
+ scale_b = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
96
+
97
+ out = ops.cutlass_scaled_sparse_mm(a,
98
+ b_comp,
99
+ e,
100
+ scale_a,
101
+ scale_b,
102
+ out_dtype=torch.bfloat16)
103
+ baseline = baseline_scaled_mm(a,
104
+ b,
105
+ scale_a,
106
+ scale_b,
107
+ out_dtype=torch.bfloat16)
108
+
109
+ torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
110
+
111
+
112
+MNK_FACTORS = [
113
+ (1, 256, 128),
114
+ (1, 16384, 1024),
115
+ (1, 24576, 512),
116
+ (16, 256, 512),
117
+ (16, 16384, 128),
118
+ (16, 24576, 4096),
119
+ (32, 8192, 4096),
120
+ (32, 16384, 4096),
121
+ (33, 1024, 1024),
122
+ (33, 8192, 128),
123
+ (64, 2048, 512),
124
+ (64, 16384, 1024),
125
+ (100, 8192, 512),
126
+ (128, 32768, 4096),
127
+ (256, 4096, 4096),
128
+ (512, 256, 1024),
129
+ (512, 8192, 4096),
130
+ (512, 16384, 128),
131
+ (512, 24576, 128),
132
+]
133
+
134
+
135
+# Test working with a subset of A and B for sparse matmul
136
+@pytest.mark.skip(reason="2of4 sparse w16a16 CUTLASS produces bad output.")
137
+@pytest.mark.skipif(not sparse_cutlass_supported(),
138
+ reason="Sparse CUTLASS is not supported on this GPU type.")
139
+@pytest.mark.parametrize("m, k, n", MNK_FACTORS)
140
+@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
141
+def test_cutlass_sparse_gemm(m: int, k: int, n: int, dtype: Type[torch.dtype]):
142
+
143
+ # Create tensors
144
+ b_comp, e, a, b = make_rand_sparse_tensors(dtype, m, n, k)
145
+ scale_a = torch.ones((1, 1), device="cuda", dtype=torch.float32)
146
+ scale_b = torch.ones((1, 1), device="cuda", dtype=torch.float32)
147
+
148
+ out = ops.cutlass_scaled_sparse_mm(a,
149
+ b_comp,
150
+ e,
151
+ scale_a,
152
+ scale_b,
153
+ out_dtype=dtype)
154
+ baseline = F.linear(a, b.T)
155
+
156
+ torch.testing.assert_close(out, baseline, rtol=1e-2, atol=1e-2)
157
+
158
+
159
+@pytest.mark.skipif(not sparse_cutlass_supported(),
160
+ reason="Sparse CUTLASS is not supported on this GPU type.")
161
+@pytest.mark.parametrize("m, k, n", MNK_FACTORS)
162
+@pytest.mark.skipif(not current_platform.has_device_capability(89),
163
+ reason="FP8 is not supported on this GPU type.")
164
+def test_cutlass_sparse_fp8_gemm(m: int, n: int, k: int):
165
+
166
+ # Create tensors
167
+ b_comp, e, a, b = make_rand_sparse_tensors(torch.float8_e4m3fn, m, n, k)
168
+ scale_a = (torch.randn((1, 1), device="cuda", dtype=torch.float32))
169
+ scale_b = (torch.randn((1, 1), device="cuda", dtype=torch.float32))
170
+
171
+ out = ops.cutlass_scaled_sparse_mm(a,
172
+ b_comp,
173
+ e,
174
+ scale_a,
175
+ scale_b,
176
+ out_dtype=torch.bfloat16)
177
+
178
+ baseline = baseline_scaled_mm(a,
179
+ b,
180
+ scale_a,
181
+ scale_b,
182
+ out_dtype=torch.bfloat16)
183
+
184
+ torch.testing.assert_close(out, baseline, rtol=1e0, atol=2e0)
185
+
186
+
187
+@pytest.mark.skipif(not sparse_cutlass_supported(),
188
+ reason="Sparse CUTLASS is not supported on this GPU type.")
189
+@pytest.mark.parametrize("m,k,n", MNK_FACTORS)
190
+@pytest.mark.parametrize("per_act_token", [True, False])
191
+@pytest.mark.parametrize("per_out_ch", [True, False])
192
+@pytest.mark.parametrize("use_bias", [True, False])
193
+def test_cutlass_sparse_int8_gemm(m: int, n: int, k: int, per_act_token: bool,
194
+ per_out_ch: bool, use_bias: bool):
195
+
196
+ # Create tensors
197
+ b_comp, e, a, b = make_rand_sparse_tensors(torch.int8, m, n, k)
198
+ scale_a = (torch.randn((1, 1), device="cuda", dtype=torch.float32))
199
+ scale_b = (torch.randn((1, 1), device="cuda", dtype=torch.float32))
200
+
201
+ out = ops.cutlass_scaled_sparse_mm(a,
202
+ b_comp,
203
+ e,
204
+ scale_a,
205
+ scale_b,
206
+ out_dtype=torch.bfloat16)
207
+
208
+ baseline = baseline_scaled_mm(a,
209
+ b,
210
+ scale_a,
211
+ scale_b,
212
+ out_dtype=torch.bfloat16)
213
+
214
+ torch.testing.assert_close(out, baseline, rtol=1e0, atol=2e0)
tests/kernels/test_semi_structured.py
deleted
-134
@@ -1,134 +0,0 @@
1
-"""Tests for sparse cutlass kernels
2
-
3
-Run `pytest tests/kernels/test_semi_structured.py`.
4
-"""
5
-from typing import Optional, Tuple, Type
6
-
7
-import pytest
8
-import torch
9
-
10
-from vllm import _custom_ops as ops
11
-from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
12
- sparse_cutlass_supported)
13
-from vllm.platforms import current_platform
14
-
15
-CUDA_DEVICES = [
16
- f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
17
-]
18
-
19
-capability = current_platform.get_device_capability()
20
-capability = capability[0] * 10 + capability[1]
21
-
22
-
23
-def to_fp8(tensor: torch.Tensor):
24
- finfo = torch.finfo(torch.float8_e4m3fn)
25
- return torch.round(tensor.clamp(
26
- min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
27
-
28
-
29
-def to_int8(tensor: torch.Tensor):
30
- return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
31
-
32
-
33
-def rand_int8(shape: tuple, device: str = "cuda"):
34
- return to_int8(torch.rand(shape, device=device) * 255 - 128)
35
-
36
-
37
-def to_bf16(tensor: torch.Tensor) -> torch.Tensor:
38
- return tensor.to(dtype=torch.bfloat16)
39
-
40
-
41
-def to_fp16(tensor: torch.Tensor) -> torch.Tensor:
42
- return tensor.to(dtype=torch.float16)
43
-
44
-
45
-def prune_to_2_4(tensor):
46
- # Reshape tensor to [N, 4] where N is number of groups of 4
47
- original_shape = tensor.shape
48
- reshaped = tensor.reshape(-1, 4)
49
-
50
- # Get indices of top 2 absolute values in each group of 4
51
- _, indices = torch.topk(torch.abs(reshaped), k=2, dim=1)
52
-
53
- # Create binary mask
54
- mask = torch.zeros_like(reshaped)
55
- mask.scatter_(dim=1,
56
- index=indices,
57
- src=torch.ones_like(indices, dtype=mask.dtype))
58
-
59
- # Apply mask and reshape back
60
- pruned = reshaped * mask
61
-
62
- # Turn all -0.0 to 0.0
63
- pruned[pruned == -0.0] = 0.0
64
-
65
- return pruned.reshape(original_shape)
66
-
67
-
68
-def make_rand_sparse_tensors(
69
- dtype: torch.dtype, m: int, n: int, k: int
70
-) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
71
- a = torch.randn((m, k), device='cuda') * 5
72
- b = torch.randn((n, k), device='cuda').t() * 5
73
-
74
- b = prune_to_2_4(b.t()).t()
75
-
76
- if dtype == torch.int8:
77
- a, b = to_int8(a), to_int8(b)
78
- elif dtype == torch.float8_e4m3fn:
79
- a, b = to_fp8(a), to_fp8(b)
80
- elif dtype == torch.float16:
81
- a, b = to_fp16(a), to_fp16(b)
82
- elif dtype == torch.bfloat16:
83
- a, b = to_bf16(a), to_bf16(b)
84
- else:
85
- raise ValueError("unsupported dtype")
86
-
87
- b_compressed, e = ops.cutlass_sparse_compress(b.t())
88
-
89
- # Compressed B, Metadata, Original A, B
90
- return b_compressed, e, a, b
91
-
92
-
93
-def baseline_scaled_mm(a: torch.Tensor,
94
- b: torch.Tensor,
95
- scale_a: torch.Tensor,
96
- scale_b: torch.Tensor,
97
- out_dtype: Type[torch.dtype],
98
- bias: Optional[torch.Tensor] = None) -> torch.Tensor:
99
- output = (scale_a * (scale_b * (torch.mm(
100
- a.to(dtype=torch.float32), b.to(dtype=torch.float32))))).to(out_dtype)
101
- if bias is not None:
102
- output = output + bias
103
-
104
- return output
105
-
106
-
107
-@pytest.mark.skipif(not sparse_cutlass_supported(),
108
- reason="Sparse FP8 is not yet supported on this GPU type.")
109
-# Test working with a subset of A and B for sparse matmul
110
-def test_cutlass_sparse_subset():
111
-
112
- big_m = 1024
113
- m, n, k = 512, 512, 512
114
-
115
- # Create tensors
116
- b_comp, e, whole_a, b = make_rand_sparse_tensors(torch.float8_e4m3fn,
117
- big_m, n, k)
118
- a = whole_a[0:m, 0:k]
119
- scale_a = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
120
- scale_b = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
121
-
122
- out = ops.cutlass_scaled_sparse_mm(a,
123
- b_comp,
124
- e,
125
- scale_a,
126
- scale_b,
127
- out_dtype=torch.bfloat16)
128
- baseline = baseline_scaled_mm(a,
129
- b,
130
- scale_a,
131
- scale_b,
132
- out_dtype=torch.bfloat16)
133
-
134
- torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
tests/kernels/utils.py
+26
-1
@@ -5,7 +5,7 @@ import random
5
import unittest
6
from numbers import Number
7
from typing import (Any, Dict, List, NamedTuple, Optional, Sequence, Tuple,
8
- Union)
8
+ Type, Union)
9
10
import pytest
11
import torch
@@ -1100,3 +1100,28 @@ def opcheck(op: Union[torch._ops.OpOverload, torch._ops.OpOverloadPacket,
1100
kwargs,
1101
test_utils=test_utils,
1102
raise_exception=raise_exception) if cond else {}
1103
+
1104
+
1105
+# For testing quantized linear kernels
1106
+def to_fp8(tensor: torch.Tensor):
1107
+ finfo = torch.finfo(torch.float8_e4m3fn)
1108
+ return torch.round(tensor.clamp(
1109
+ min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
1110
+
1111
+
1112
+def to_int8(tensor: torch.Tensor):
1113
+ return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
1114
+
1115
+
1116
+def baseline_scaled_mm(a: torch.Tensor,
1117
+ b: torch.Tensor,
1118
+ scale_a: torch.Tensor,
1119
+ scale_b: torch.Tensor,
1120
+ out_dtype: Type[torch.dtype],
1121
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
1122
+ output = (scale_a * (scale_b * (torch.mm(
1123
+ a.to(dtype=torch.float32), b.to(dtype=torch.float32))))).to(out_dtype)
1124
+ if bias is not None:
1125
+ output = output + bias
1126
+
1127
+ return output
tests/quantization/test_compressed_tensors.py
+3
-1
@@ -313,8 +313,10 @@ def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):
313
assert output
314
315
316
+@pytest.mark.skip(reason="2of4 sparse w16a16 CUTLASS produces bad output.")
317
@pytest.mark.skipif(not sparse_cutlass_supported(),
317
- reason="Sparse FP8 is not yet supported on this GPU type.")
318
+ reason="2of4 Sparse is not yet supported on this GPU type."
319
+ )
320
@pytest.mark.parametrize(
321
"args_2of4",
322
[("nm-testing/TinyLlama-1.1B-Chat-v1.0-2of4-Sparse-Dense-Compressor")])
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+17
-8
@@ -9,6 +9,7 @@ from compressed_tensors.quantization import (QuantizationArgs,
9
QuantizationType)
10
from pydantic import BaseModel
11
12
+from vllm.logger import init_logger
13
from vllm.model_executor.layers.fused_moe import FusedMoE
14
from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
15
UnquantizedLinearMethod)
@@ -27,6 +28,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
28
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
29
from vllm.platforms import current_platform
30
31
+logger = init_logger(__name__)
32
+
33
__all__ = ["CompressedTensorsLinearMethod"]
34
35
SPARSITY_CONFIG_NAME: Literal["sparsity_config"] = "sparsity_config"
@@ -79,6 +82,8 @@ class CompressedTensorsConfig(QuantizationConfig):
82
return UnquantizedLinearMethod()
83
if isinstance(layer, LinearBase):
84
scheme = self.get_scheme(layer=layer, layer_name=prefix)
85
+ if scheme is None:
86
+ return UnquantizedLinearMethod()
87
layer.scheme = scheme
88
return CompressedTensorsLinearMethod(self)
89
if isinstance(layer, Attention):
@@ -340,10 +345,10 @@ class CompressedTensorsConfig(QuantizationConfig):
345
raise NotImplementedError(
346
"No compressed-tensors compatible scheme was found.")
347
343
- def get_scheme(
344
- self,
345
- layer: torch.nn.Module,
346
- layer_name: Optional[str] = None) -> "CompressedTensorsScheme":
348
+ def get_scheme(self,
349
+ layer: torch.nn.Module,
350
+ layer_name: Optional[str] = None
351
+ ) -> Optional["CompressedTensorsScheme"]:
352
"""
353
compressed-tensors supports non uniform in the following way:
354
@@ -353,10 +358,7 @@ class CompressedTensorsConfig(QuantizationConfig):
358
which can be a full layer_name, a regex for a layer_name, or
359
an nn.Module name.
360
356
- We first check whether a layer is in the ignore group and use
357
- CompressedTensorsUnquantized (i.e. fp16/bf16) scheme for the layer
358
-
359
- We then detect whether a layer_name is found in any target and
361
+ Detect whether a layer_name is found in any target and
362
use the quantization scheme corresponding to the matched target
363
to select the CompressedTensorsScheme used for infernece.
364
"""
@@ -394,6 +396,13 @@ class CompressedTensorsConfig(QuantizationConfig):
396
if self.supports_cutlass_24(weight_quant=weight_quant,
397
input_quant=input_quant,
398
sparsity_scheme=sparsity_scheme):
399
+ # FIXME(tlrmchlsmth): layers using W16A16 CUTLASS 2:4 sparse kernels
400
+ # currently produce bad output in some cases
401
+ if weight_quant is None:
402
+ logger.warning_once(
403
+ "CompressedTensors24 scheme is disabled for the w16a16 "
404
+ "case. Falling back to UnquantizedLinearMethod")
405
+ return None
406
# Have a valid sparsity scheme
407
# Validate layer is supported by Cutlass 2:4 Kernel
408
scheme = CompressedTensors24(quantized=weight_quant is not None