@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aa2cd2c43d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Bugfix] Disable w16a16 2of4 sparse CompressedTensors24 (#12417)

Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com> Co-authored-by: mgoin <michael@neuralmagic.com>

Tyler Michael Smith committed Jan 26, 2025 at 06:59 UTC aa2cd2c43d1d19ece0f3b36ad716c3a9b8a2def0
6 files changed +263 -169
tests/kernels/test_cutlass.py
+3 -25
@@ -2,7 +2,7 @@
2
3 Run `pytest tests/kernels/test_cutlass.py`.
4 """
5 -from typing import Optional, Type
5 +from typing import Type
6
7 import pytest
8 import torch
@@ -11,6 +11,8 @@ from tests.kernels.utils import opcheck
11 from vllm import _custom_ops as ops
12 from vllm.platforms import current_platform
13
14 +from .utils import baseline_scaled_mm, to_fp8, to_int8
15 +
16 MNK_FACTORS = [
17 (1, 256, 128),
18 (1, 16384, 1024),
@@ -41,34 +43,10 @@ capability = current_platform.get_device_capability()
43 capability = capability[0] * 10 + capability[1]
44
45
44 -def to_fp8(tensor: torch.Tensor):
45 - finfo = torch.finfo(torch.float8_e4m3fn)
46 - return torch.round(tensor.clamp(
47 - min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
48 -
49 -
50 -def to_int8(tensor: torch.Tensor):
51 - return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
52 -
53 -
46 def rand_int8(shape: tuple, device: str = "cuda"):
47 return to_int8(torch.rand(shape, device=device) * 255 - 128)
48
49
58 -def baseline_scaled_mm(a: torch.Tensor,
59 - b: torch.Tensor,
60 - scale_a: torch.Tensor,
61 - scale_b: torch.Tensor,
62 - out_dtype: Type[torch.dtype],
63 - bias: Optional[torch.Tensor] = None) -> torch.Tensor:
64 - output = (scale_a * (scale_b * (torch.mm(
65 - a.to(dtype=torch.float32), b.to(dtype=torch.float32))))).to(out_dtype)
66 - if bias is not None:
67 - output = output + bias
68 -
69 - return output
70 -
71 -
50 def cutlass_fp8_gemm_helper(m: int,
51 n: int,
52 k: int,
tests/kernels/test_cutlass_2of4_sparse.py new
+214
@@ -0,0 +1,214 @@
1 +"""Tests for sparse cutlass kernels
2 +
3 +Run `pytest tests/kernels/test_semi_structured.py`.
4 +"""
5 +from typing import Tuple, Type
6 +
7 +import pytest
8 +import torch
9 +import torch.nn.functional as F
10 +
11 +from vllm import _custom_ops as ops
12 +from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
13 + sparse_cutlass_supported)
14 +from vllm.platforms import current_platform
15 +
16 +from .utils import baseline_scaled_mm, to_fp8, to_int8
17 +
18 +CUDA_DEVICES = [
19 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
20 +]
21 +
22 +capability = current_platform.get_device_capability()
23 +capability = capability[0] * 10 + capability[1]
24 +
25 +
26 +def to_bf16(tensor: torch.Tensor) -> torch.Tensor:
27 + return tensor.to(dtype=torch.bfloat16)
28 +
29 +
30 +def to_fp16(tensor: torch.Tensor) -> torch.Tensor:
31 + return tensor.to(dtype=torch.float16)
32 +
33 +
34 +def prune_to_2_4(tensor):
35 + # Reshape tensor to [N, 4] where N is number of groups of 4
36 + original_shape = tensor.shape
37 + reshaped = tensor.reshape(-1, 4)
38 +
39 + # Get indices of top 2 absolute values in each group of 4
40 + _, indices = torch.topk(torch.abs(reshaped), k=2, dim=1)
41 +
42 + # Create binary mask
43 + mask = torch.zeros_like(reshaped)
44 + mask.scatter_(dim=1,
45 + index=indices,
46 + src=torch.ones_like(indices, dtype=mask.dtype))
47 +
48 + # Apply mask and reshape back
49 + pruned = reshaped * mask
50 +
51 + # Turn all -0.0 to 0.0
52 + pruned[pruned == -0.0] = 0.0
53 +
54 + return pruned.reshape(original_shape)
55 +
56 +
57 +def make_rand_sparse_tensors(
58 + dtype: torch.dtype, m: int, n: int, k: int
59 +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
60 + a = torch.randn((m, k), device='cuda') * 5
61 + b = torch.randn((n, k), device='cuda').t() * 5
62 +
63 + b = prune_to_2_4(b.t()).t()
64 +
65 + if dtype == torch.int8:
66 + a, b = to_int8(a), to_int8(b)
67 + elif dtype == torch.float8_e4m3fn:
68 + a, b = to_fp8(a), to_fp8(b)
69 + elif dtype == torch.float16:
70 + a, b = to_fp16(a), to_fp16(b)
71 + elif dtype == torch.bfloat16:
72 + a, b = to_bf16(a), to_bf16(b)
73 + else:
74 + raise ValueError("unsupported dtype")
75 +
76 + b_compressed, e = ops.cutlass_sparse_compress(b.t())
77 +
78 + # Compressed B, Metadata, Original A, B
79 + return b_compressed, e, a, b
80 +
81 +
82 +@pytest.mark.skipif(not sparse_cutlass_supported(),
83 + reason="Sparse CUTLASS is not supported on this GPU type.")
84 +# Test working with a subset of A and B for sparse matmul
85 +def test_cutlass_sparse_subset():
86 +
87 + big_m = 1024
88 + m, n, k = 512, 512, 512
89 +
90 + # Create tensors
91 + b_comp, e, whole_a, b = make_rand_sparse_tensors(torch.float8_e4m3fn,
92 + big_m, n, k)
93 + a = whole_a[0:m, 0:k]
94 + scale_a = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
95 + scale_b = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
96 +
97 + out = ops.cutlass_scaled_sparse_mm(a,
98 + b_comp,
99 + e,
100 + scale_a,
101 + scale_b,
102 + out_dtype=torch.bfloat16)
103 + baseline = baseline_scaled_mm(a,
104 + b,
105 + scale_a,
106 + scale_b,
107 + out_dtype=torch.bfloat16)
108 +
109 + torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
110 +
111 +
112 +MNK_FACTORS = [
113 + (1, 256, 128),
114 + (1, 16384, 1024),
115 + (1, 24576, 512),
116 + (16, 256, 512),
117 + (16, 16384, 128),
118 + (16, 24576, 4096),
119 + (32, 8192, 4096),
120 + (32, 16384, 4096),
121 + (33, 1024, 1024),
122 + (33, 8192, 128),
123 + (64, 2048, 512),
124 + (64, 16384, 1024),
125 + (100, 8192, 512),
126 + (128, 32768, 4096),
127 + (256, 4096, 4096),
128 + (512, 256, 1024),
129 + (512, 8192, 4096),
130 + (512, 16384, 128),
131 + (512, 24576, 128),
132 +]
133 +
134 +
135 +# Test working with a subset of A and B for sparse matmul
136 +@pytest.mark.skip(reason="2of4 sparse w16a16 CUTLASS produces bad output.")
137 +@pytest.mark.skipif(not sparse_cutlass_supported(),
138 + reason="Sparse CUTLASS is not supported on this GPU type.")
139 +@pytest.mark.parametrize("m, k, n", MNK_FACTORS)
140 +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
141 +def test_cutlass_sparse_gemm(m: int, k: int, n: int, dtype: Type[torch.dtype]):
142 +
143 + # Create tensors
144 + b_comp, e, a, b = make_rand_sparse_tensors(dtype, m, n, k)
145 + scale_a = torch.ones((1, 1), device="cuda", dtype=torch.float32)
146 + scale_b = torch.ones((1, 1), device="cuda", dtype=torch.float32)
147 +
148 + out = ops.cutlass_scaled_sparse_mm(a,
149 + b_comp,
150 + e,
151 + scale_a,
152 + scale_b,
153 + out_dtype=dtype)
154 + baseline = F.linear(a, b.T)
155 +
156 + torch.testing.assert_close(out, baseline, rtol=1e-2, atol=1e-2)
157 +
158 +
159 +@pytest.mark.skipif(not sparse_cutlass_supported(),
160 + reason="Sparse CUTLASS is not supported on this GPU type.")
161 +@pytest.mark.parametrize("m, k, n", MNK_FACTORS)
162 +@pytest.mark.skipif(not current_platform.has_device_capability(89),
163 + reason="FP8 is not supported on this GPU type.")
164 +def test_cutlass_sparse_fp8_gemm(m: int, n: int, k: int):
165 +
166 + # Create tensors
167 + b_comp, e, a, b = make_rand_sparse_tensors(torch.float8_e4m3fn, m, n, k)
168 + scale_a = (torch.randn((1, 1), device="cuda", dtype=torch.float32))
169 + scale_b = (torch.randn((1, 1), device="cuda", dtype=torch.float32))
170 +
171 + out = ops.cutlass_scaled_sparse_mm(a,
172 + b_comp,
173 + e,
174 + scale_a,
175 + scale_b,
176 + out_dtype=torch.bfloat16)
177 +
178 + baseline = baseline_scaled_mm(a,
179 + b,
180 + scale_a,
181 + scale_b,
182 + out_dtype=torch.bfloat16)
183 +
184 + torch.testing.assert_close(out, baseline, rtol=1e0, atol=2e0)
185 +
186 +
187 +@pytest.mark.skipif(not sparse_cutlass_supported(),
188 + reason="Sparse CUTLASS is not supported on this GPU type.")
189 +@pytest.mark.parametrize("m,k,n", MNK_FACTORS)
190 +@pytest.mark.parametrize("per_act_token", [True, False])
191 +@pytest.mark.parametrize("per_out_ch", [True, False])
192 +@pytest.mark.parametrize("use_bias", [True, False])
193 +def test_cutlass_sparse_int8_gemm(m: int, n: int, k: int, per_act_token: bool,
194 + per_out_ch: bool, use_bias: bool):
195 +
196 + # Create tensors
197 + b_comp, e, a, b = make_rand_sparse_tensors(torch.int8, m, n, k)
198 + scale_a = (torch.randn((1, 1), device="cuda", dtype=torch.float32))
199 + scale_b = (torch.randn((1, 1), device="cuda", dtype=torch.float32))
200 +
201 + out = ops.cutlass_scaled_sparse_mm(a,
202 + b_comp,
203 + e,
204 + scale_a,
205 + scale_b,
206 + out_dtype=torch.bfloat16)
207 +
208 + baseline = baseline_scaled_mm(a,
209 + b,
210 + scale_a,
211 + scale_b,
212 + out_dtype=torch.bfloat16)
213 +
214 + torch.testing.assert_close(out, baseline, rtol=1e0, atol=2e0)
tests/kernels/test_semi_structured.py deleted
-134
@@ -1,134 +0,0 @@
1 -"""Tests for sparse cutlass kernels
2 -
3 -Run `pytest tests/kernels/test_semi_structured.py`.
4 -"""
5 -from typing import Optional, Tuple, Type
6 -
7 -import pytest
8 -import torch
9 -
10 -from vllm import _custom_ops as ops
11 -from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
12 - sparse_cutlass_supported)
13 -from vllm.platforms import current_platform
14 -
15 -CUDA_DEVICES = [
16 - f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
17 -]
18 -
19 -capability = current_platform.get_device_capability()
20 -capability = capability[0] * 10 + capability[1]
21 -
22 -
23 -def to_fp8(tensor: torch.Tensor):
24 - finfo = torch.finfo(torch.float8_e4m3fn)
25 - return torch.round(tensor.clamp(
26 - min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
27 -
28 -
29 -def to_int8(tensor: torch.Tensor):
30 - return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
31 -
32 -
33 -def rand_int8(shape: tuple, device: str = "cuda"):
34 - return to_int8(torch.rand(shape, device=device) * 255 - 128)
35 -
36 -
37 -def to_bf16(tensor: torch.Tensor) -> torch.Tensor:
38 - return tensor.to(dtype=torch.bfloat16)
39 -
40 -
41 -def to_fp16(tensor: torch.Tensor) -> torch.Tensor:
42 - return tensor.to(dtype=torch.float16)
43 -
44 -
45 -def prune_to_2_4(tensor):
46 - # Reshape tensor to [N, 4] where N is number of groups of 4
47 - original_shape = tensor.shape
48 - reshaped = tensor.reshape(-1, 4)
49 -
50 - # Get indices of top 2 absolute values in each group of 4
51 - _, indices = torch.topk(torch.abs(reshaped), k=2, dim=1)
52 -
53 - # Create binary mask
54 - mask = torch.zeros_like(reshaped)
55 - mask.scatter_(dim=1,
56 - index=indices,
57 - src=torch.ones_like(indices, dtype=mask.dtype))
58 -
59 - # Apply mask and reshape back
60 - pruned = reshaped * mask
61 -
62 - # Turn all -0.0 to 0.0
63 - pruned[pruned == -0.0] = 0.0
64 -
65 - return pruned.reshape(original_shape)
66 -
67 -
68 -def make_rand_sparse_tensors(
69 - dtype: torch.dtype, m: int, n: int, k: int
70 -) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
71 - a = torch.randn((m, k), device='cuda') * 5
72 - b = torch.randn((n, k), device='cuda').t() * 5
73 -
74 - b = prune_to_2_4(b.t()).t()
75 -
76 - if dtype == torch.int8:
77 - a, b = to_int8(a), to_int8(b)
78 - elif dtype == torch.float8_e4m3fn:
79 - a, b = to_fp8(a), to_fp8(b)
80 - elif dtype == torch.float16:
81 - a, b = to_fp16(a), to_fp16(b)
82 - elif dtype == torch.bfloat16:
83 - a, b = to_bf16(a), to_bf16(b)
84 - else:
85 - raise ValueError("unsupported dtype")
86 -
87 - b_compressed, e = ops.cutlass_sparse_compress(b.t())
88 -
89 - # Compressed B, Metadata, Original A, B
90 - return b_compressed, e, a, b
91 -
92 -
93 -def baseline_scaled_mm(a: torch.Tensor,
94 - b: torch.Tensor,
95 - scale_a: torch.Tensor,
96 - scale_b: torch.Tensor,
97 - out_dtype: Type[torch.dtype],
98 - bias: Optional[torch.Tensor] = None) -> torch.Tensor:
99 - output = (scale_a * (scale_b * (torch.mm(
100 - a.to(dtype=torch.float32), b.to(dtype=torch.float32))))).to(out_dtype)
101 - if bias is not None:
102 - output = output + bias
103 -
104 - return output
105 -
106 -
107 -@pytest.mark.skipif(not sparse_cutlass_supported(),
108 - reason="Sparse FP8 is not yet supported on this GPU type.")
109 -# Test working with a subset of A and B for sparse matmul
110 -def test_cutlass_sparse_subset():
111 -
112 - big_m = 1024
113 - m, n, k = 512, 512, 512
114 -
115 - # Create tensors
116 - b_comp, e, whole_a, b = make_rand_sparse_tensors(torch.float8_e4m3fn,
117 - big_m, n, k)
118 - a = whole_a[0:m, 0:k]
119 - scale_a = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
120 - scale_b = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
121 -
122 - out = ops.cutlass_scaled_sparse_mm(a,
123 - b_comp,
124 - e,
125 - scale_a,
126 - scale_b,
127 - out_dtype=torch.bfloat16)
128 - baseline = baseline_scaled_mm(a,
129 - b,
130 - scale_a,
131 - scale_b,
132 - out_dtype=torch.bfloat16)
133 -
134 - torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
tests/kernels/utils.py
+26 -1
@@ -5,7 +5,7 @@ import random
5 import unittest
6 from numbers import Number
7 from typing import (Any, Dict, List, NamedTuple, Optional, Sequence, Tuple,
8 - Union)
8 + Type, Union)
9
10 import pytest
11 import torch
@@ -1100,3 +1100,28 @@ def opcheck(op: Union[torch._ops.OpOverload, torch._ops.OpOverloadPacket,
1100 kwargs,
1101 test_utils=test_utils,
1102 raise_exception=raise_exception) if cond else {}
1103 +
1104 +
1105 +# For testing quantized linear kernels
1106 +def to_fp8(tensor: torch.Tensor):
1107 + finfo = torch.finfo(torch.float8_e4m3fn)
1108 + return torch.round(tensor.clamp(
1109 + min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
1110 +
1111 +
1112 +def to_int8(tensor: torch.Tensor):
1113 + return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
1114 +
1115 +
1116 +def baseline_scaled_mm(a: torch.Tensor,
1117 + b: torch.Tensor,
1118 + scale_a: torch.Tensor,
1119 + scale_b: torch.Tensor,
1120 + out_dtype: Type[torch.dtype],
1121 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
1122 + output = (scale_a * (scale_b * (torch.mm(
1123 + a.to(dtype=torch.float32), b.to(dtype=torch.float32))))).to(out_dtype)
1124 + if bias is not None:
1125 + output = output + bias
1126 +
1127 + return output
tests/quantization/test_compressed_tensors.py
+3 -1
@@ -313,8 +313,10 @@ def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):
313 assert output
314
315
316 +@pytest.mark.skip(reason="2of4 sparse w16a16 CUTLASS produces bad output.")
317 @pytest.mark.skipif(not sparse_cutlass_supported(),
317 - reason="Sparse FP8 is not yet supported on this GPU type.")
318 + reason="2of4 Sparse is not yet supported on this GPU type."
319 + )
320 @pytest.mark.parametrize(
321 "args_2of4",
322 [("nm-testing/TinyLlama-1.1B-Chat-v1.0-2of4-Sparse-Dense-Compressor")])
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+17 -8
@@ -9,6 +9,7 @@ from compressed_tensors.quantization import (QuantizationArgs,
9 QuantizationType)
10 from pydantic import BaseModel
11
12 +from vllm.logger import init_logger
13 from vllm.model_executor.layers.fused_moe import FusedMoE
14 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
15 UnquantizedLinearMethod)
@@ -27,6 +28,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
28 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
29 from vllm.platforms import current_platform
30
31 +logger = init_logger(__name__)
32 +
33 __all__ = ["CompressedTensorsLinearMethod"]
34
35 SPARSITY_CONFIG_NAME: Literal["sparsity_config"] = "sparsity_config"
@@ -79,6 +82,8 @@ class CompressedTensorsConfig(QuantizationConfig):
82 return UnquantizedLinearMethod()
83 if isinstance(layer, LinearBase):
84 scheme = self.get_scheme(layer=layer, layer_name=prefix)
85 + if scheme is None:
86 + return UnquantizedLinearMethod()
87 layer.scheme = scheme
88 return CompressedTensorsLinearMethod(self)
89 if isinstance(layer, Attention):
@@ -340,10 +345,10 @@ class CompressedTensorsConfig(QuantizationConfig):
345 raise NotImplementedError(
346 "No compressed-tensors compatible scheme was found.")
347
343 - def get_scheme(
344 - self,
345 - layer: torch.nn.Module,
346 - layer_name: Optional[str] = None) -> "CompressedTensorsScheme":
348 + def get_scheme(self,
349 + layer: torch.nn.Module,
350 + layer_name: Optional[str] = None
351 + ) -> Optional["CompressedTensorsScheme"]:
352 """
353 compressed-tensors supports non uniform in the following way:
354
@@ -353,10 +358,7 @@ class CompressedTensorsConfig(QuantizationConfig):
358 which can be a full layer_name, a regex for a layer_name, or
359 an nn.Module name.
360
356 - We first check whether a layer is in the ignore group and use
357 - CompressedTensorsUnquantized (i.e. fp16/bf16) scheme for the layer
358 -
359 - We then detect whether a layer_name is found in any target and
361 + Detect whether a layer_name is found in any target and
362 use the quantization scheme corresponding to the matched target
363 to select the CompressedTensorsScheme used for infernece.
364 """
@@ -394,6 +396,13 @@ class CompressedTensorsConfig(QuantizationConfig):
396 if self.supports_cutlass_24(weight_quant=weight_quant,
397 input_quant=input_quant,
398 sparsity_scheme=sparsity_scheme):
399 + # FIXME(tlrmchlsmth): layers using W16A16 CUTLASS 2:4 sparse kernels
400 + # currently produce bad output in some cases
401 + if weight_quant is None:
402 + logger.warning_once(
403 + "CompressedTensors24 scheme is disabled for the w16a16 "
404 + "case. Falling back to UnquantizedLinearMethod")
405 + return None
406 # Have a valid sparsity scheme
407 # Validate layer is supported by Cutlass 2:4 Kernel
408 scheme = CompressedTensors24(quantized=weight_quant is not None