A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Misc] Update `w4a16` `compressed-tensors` support to include `w8a16` (#5794)
Dipika Sikka committed
Jun 25, 2024 at 15:23 UTC
dd248f76756adba4a1637b882e79ab639f957feb
5 files changed
+36
-26
tests/quantization/test_compressed_tensors.py
+12
-11
@@ -8,9 +8,9 @@ import torch
8
9
from vllm import SamplingParams
10
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
11
- CompressedTensorsLinearMethod, CompressedTensorsW4A16,
12
- CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8DynamicToken,
13
- CompressedTensorsW8A8StaticTensor)
11
+ CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
12
+ CompressedTensorsW8A8DynamicToken, CompressedTensorsW8A8StaticTensor,
13
+ CompressedTensorsWNA16)
14
15
16
@pytest.mark.parametrize("model_args", [
@@ -74,26 +74,27 @@ def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner, model_args):
74
assert qkv_proj.weight.dtype is torch.int8
75
76
77
-@pytest.mark.parametrize("w4a16_args", [
78
- ("nm-testing/tinyllama-oneshot-w4a16-channel-v2", "channel", None),
79
- ("nm-testing/tinyllama-oneshot-w4a16-group128-v2", "group", 128),
80
-])
81
-def test_compressed_tensors_w4a16(vllm_runner, w4a16_args):
82
- model, strategy, group = w4a16_args
77
+@pytest.mark.parametrize(
78
+ "wNa16_args",
79
+ [("nm-testing/tinyllama-oneshot-w4a16-channel-v2", "channel", None, 8),
80
+ ("nm-testing/tinyllama-oneshot-w4a16-group128-v2", "group", 128, 8),
81
+ ("nm-testing/tinyllama-oneshot-w8a16-per-channel", "channel", None, 4)])
82
+def test_compressed_tensors_w4a16(vllm_runner, wNa16_args):
83
+ model, strategy, group, pack_factor = wNa16_args
84
with vllm_runner(model) as llm:
85
model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
86
layer = model.model.layers[0]
87
88
qkv_proj = layer.self_attn.qkv_proj
89
assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
89
- assert isinstance(qkv_proj.scheme, CompressedTensorsW4A16)
90
+ assert isinstance(qkv_proj.scheme, CompressedTensorsWNA16)
91
92
assert qkv_proj.scheme.strategy == strategy
93
assert qkv_proj.scheme.group_size == group
94
95
assert qkv_proj.weight_packed.dtype is torch.int32
96
assert qkv_proj.weight_scale.dtype is torch.float16
96
- assert qkv_proj.weight_packed.pack_factor == 8
97
+ assert qkv_proj.weight_packed.pack_factor == pack_factor
98
99
100
def test_compressed_tensors_w4a16_marlin24(vllm_runner):
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+17
-11
@@ -7,9 +7,10 @@ from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
7
from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
8
QuantizationConfig)
9
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10
- CompressedTensorsScheme, CompressedTensorsW4A16,
11
- CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8DynamicToken,
12
- CompressedTensorsW8A8StaticTensor)
10
+ W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
11
+ CompressedTensorsScheme, CompressedTensorsW4A16Sparse24,
12
+ CompressedTensorsW8A8DynamicToken, CompressedTensorsW8A8StaticTensor,
13
+ CompressedTensorsWNA16)
14
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15
CompressionFormat, QuantizationArgs, QuantizationStrategy,
16
find_first_name_or_class_match)
@@ -108,26 +109,31 @@ class CompressedTensorsConfig(QuantizationConfig):
109
110
return is_8_bits and is_token and is_symmetric and is_dynamic
111
111
- def _is_w4a16(self, weight_quant: BaseModel,
112
- input_quant: BaseModel) -> bool:
112
+ def _is_wNa16_group_channel(self, weight_quant: BaseModel,
113
+ input_quant: BaseModel) -> bool:
114
input_quant_none = input_quant is None
114
- is_4_bits = weight_quant.num_bits == 4
115
is_symmetric = weight_quant.symmetric
116
+ is_channel_group = (
117
+ weight_quant.strategy == QuantizationStrategy.CHANNEL.value
118
+ or weight_quant.strategy == QuantizationStrategy.GROUP.value)
119
is_static = not weight_quant.dynamic
120
118
- return is_4_bits and input_quant_none and is_symmetric and is_static
121
+ return (is_channel_group and input_quant_none and is_symmetric
122
+ and is_static)
123
124
def _get_schema(self, weight_quant: BaseModel,
125
input_quant: BaseModel) -> "CompressedTensorsScheme":
126
123
- if self._is_w4a16(weight_quant, input_quant):
124
- if self.quant_format == CompressionFormat.marlin_24.value:
127
+ if self._is_wNa16_group_channel(weight_quant, input_quant):
128
+ if (self.quant_format == CompressionFormat.marlin_24.value
129
+ and weight_quant.num_bits in W4A16SPARSE24_SUPPORTED_BITS):
130
return CompressedTensorsW4A16Sparse24(
131
strategy=weight_quant.strategy,
132
num_bits=weight_quant.num_bits,
133
group_size=weight_quant.group_size)
129
- if self.quant_format == CompressionFormat.pack_quantized.value:
130
- return CompressedTensorsW4A16(
134
+ if (self.quant_format == CompressionFormat.pack_quantized.value
135
+ and weight_quant.num_bits in WNA16_SUPPORTED_BITS):
136
+ return CompressedTensorsWNA16(
137
num_bits=weight_quant.num_bits,
138
strategy=weight_quant.strategy,
139
group_size=weight_quant.group_size)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
+3
-2
@@ -1,10 +1,11 @@
1
from .compressed_tensors_scheme import CompressedTensorsScheme # noqa: F401
2
from .compressed_tensors_unquantized import ( # noqa: F401
3
CompressedTensorsUnquantized)
4
-from .compressed_tensors_w4a16 import CompressedTensorsW4A16 # noqa: F401
4
from .compressed_tensors_w4a16_24 import ( # noqa: F401
6
- CompressedTensorsW4A16Sparse24)
5
+ W4A16SPARSE24_SUPPORTED_BITS, CompressedTensorsW4A16Sparse24)
6
from .compressed_tensors_w8a8_dynamictoken import ( # noqa: F401, E501
7
CompressedTensorsW8A8DynamicToken)
8
from .compressed_tensors_w8a8_statictensor import ( # noqa: F401, E501
9
CompressedTensorsW8A8StaticTensor)
10
+from .compressed_tensors_wNa16 import WNA16_SUPPORTED_BITS # noqa: F401
11
+from .compressed_tensors_wNa16 import CompressedTensorsWNA16 # noqa: F401
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+1
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
11
from vllm.model_executor.utils import set_weight_attrs
12
13
__all__ = ["CompressedTensorsW4A16Sparse24"]
14
+W4A16SPARSE24_SUPPORTED_BITS = [4]
15
16
17
class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
renamed
+3
-2
@@ -11,10 +11,11 @@ from vllm.model_executor.layers.quantization.gptq_marlin import (
11
marlin_permute_scales)
12
from vllm.model_executor.utils import set_weight_attrs
13
14
-__all__ = ["CompressedTensorsW4A16"]
14
+__all__ = ["CompressedTensorsWNA16"]
15
+WNA16_SUPPORTED_BITS = [4, 8]
16
17
17
-class CompressedTensorsW4A16(CompressedTensorsScheme):
18
+class CompressedTensorsWNA16(CompressedTensorsScheme):
19
20
def __init__(self,
21
strategy: str,