@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dd248f7675
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Misc] Update `w4a16` `compressed-tensors` support to include `w8a16` (#5794)

Dipika Sikka committed Jun 25, 2024 at 15:23 UTC dd248f76756adba4a1637b882e79ab639f957feb
5 files changed +36 -26
tests/quantization/test_compressed_tensors.py
+12 -11
@@ -8,9 +8,9 @@ import torch
8
9 from vllm import SamplingParams
10 from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
11 - CompressedTensorsLinearMethod, CompressedTensorsW4A16,
12 - CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8DynamicToken,
13 - CompressedTensorsW8A8StaticTensor)
11 + CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
12 + CompressedTensorsW8A8DynamicToken, CompressedTensorsW8A8StaticTensor,
13 + CompressedTensorsWNA16)
14
15
16 @pytest.mark.parametrize("model_args", [
@@ -74,26 +74,27 @@ def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner, model_args):
74 assert qkv_proj.weight.dtype is torch.int8
75
76
77 -@pytest.mark.parametrize("w4a16_args", [
78 - ("nm-testing/tinyllama-oneshot-w4a16-channel-v2", "channel", None),
79 - ("nm-testing/tinyllama-oneshot-w4a16-group128-v2", "group", 128),
80 -])
81 -def test_compressed_tensors_w4a16(vllm_runner, w4a16_args):
82 - model, strategy, group = w4a16_args
77 +@pytest.mark.parametrize(
78 + "wNa16_args",
79 + [("nm-testing/tinyllama-oneshot-w4a16-channel-v2", "channel", None, 8),
80 + ("nm-testing/tinyllama-oneshot-w4a16-group128-v2", "group", 128, 8),
81 + ("nm-testing/tinyllama-oneshot-w8a16-per-channel", "channel", None, 4)])
82 +def test_compressed_tensors_w4a16(vllm_runner, wNa16_args):
83 + model, strategy, group, pack_factor = wNa16_args
84 with vllm_runner(model) as llm:
85 model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
86 layer = model.model.layers[0]
87
88 qkv_proj = layer.self_attn.qkv_proj
89 assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
89 - assert isinstance(qkv_proj.scheme, CompressedTensorsW4A16)
90 + assert isinstance(qkv_proj.scheme, CompressedTensorsWNA16)
91
92 assert qkv_proj.scheme.strategy == strategy
93 assert qkv_proj.scheme.group_size == group
94
95 assert qkv_proj.weight_packed.dtype is torch.int32
96 assert qkv_proj.weight_scale.dtype is torch.float16
96 - assert qkv_proj.weight_packed.pack_factor == 8
97 + assert qkv_proj.weight_packed.pack_factor == pack_factor
98
99
100 def test_compressed_tensors_w4a16_marlin24(vllm_runner):
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+17 -11
@@ -7,9 +7,10 @@ from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
7 from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
8 QuantizationConfig)
9 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10 - CompressedTensorsScheme, CompressedTensorsW4A16,
11 - CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8DynamicToken,
12 - CompressedTensorsW8A8StaticTensor)
10 + W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
11 + CompressedTensorsScheme, CompressedTensorsW4A16Sparse24,
12 + CompressedTensorsW8A8DynamicToken, CompressedTensorsW8A8StaticTensor,
13 + CompressedTensorsWNA16)
14 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15 CompressionFormat, QuantizationArgs, QuantizationStrategy,
16 find_first_name_or_class_match)
@@ -108,26 +109,31 @@ class CompressedTensorsConfig(QuantizationConfig):
109
110 return is_8_bits and is_token and is_symmetric and is_dynamic
111
111 - def _is_w4a16(self, weight_quant: BaseModel,
112 - input_quant: BaseModel) -> bool:
112 + def _is_wNa16_group_channel(self, weight_quant: BaseModel,
113 + input_quant: BaseModel) -> bool:
114 input_quant_none = input_quant is None
114 - is_4_bits = weight_quant.num_bits == 4
115 is_symmetric = weight_quant.symmetric
116 + is_channel_group = (
117 + weight_quant.strategy == QuantizationStrategy.CHANNEL.value
118 + or weight_quant.strategy == QuantizationStrategy.GROUP.value)
119 is_static = not weight_quant.dynamic
120
118 - return is_4_bits and input_quant_none and is_symmetric and is_static
121 + return (is_channel_group and input_quant_none and is_symmetric
122 + and is_static)
123
124 def _get_schema(self, weight_quant: BaseModel,
125 input_quant: BaseModel) -> "CompressedTensorsScheme":
126
123 - if self._is_w4a16(weight_quant, input_quant):
124 - if self.quant_format == CompressionFormat.marlin_24.value:
127 + if self._is_wNa16_group_channel(weight_quant, input_quant):
128 + if (self.quant_format == CompressionFormat.marlin_24.value
129 + and weight_quant.num_bits in W4A16SPARSE24_SUPPORTED_BITS):
130 return CompressedTensorsW4A16Sparse24(
131 strategy=weight_quant.strategy,
132 num_bits=weight_quant.num_bits,
133 group_size=weight_quant.group_size)
129 - if self.quant_format == CompressionFormat.pack_quantized.value:
130 - return CompressedTensorsW4A16(
134 + if (self.quant_format == CompressionFormat.pack_quantized.value
135 + and weight_quant.num_bits in WNA16_SUPPORTED_BITS):
136 + return CompressedTensorsWNA16(
137 num_bits=weight_quant.num_bits,
138 strategy=weight_quant.strategy,
139 group_size=weight_quant.group_size)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
+3 -2
@@ -1,10 +1,11 @@
1 from .compressed_tensors_scheme import CompressedTensorsScheme # noqa: F401
2 from .compressed_tensors_unquantized import ( # noqa: F401
3 CompressedTensorsUnquantized)
4 -from .compressed_tensors_w4a16 import CompressedTensorsW4A16 # noqa: F401
4 from .compressed_tensors_w4a16_24 import ( # noqa: F401
6 - CompressedTensorsW4A16Sparse24)
5 + W4A16SPARSE24_SUPPORTED_BITS, CompressedTensorsW4A16Sparse24)
6 from .compressed_tensors_w8a8_dynamictoken import ( # noqa: F401, E501
7 CompressedTensorsW8A8DynamicToken)
8 from .compressed_tensors_w8a8_statictensor import ( # noqa: F401, E501
9 CompressedTensorsW8A8StaticTensor)
10 +from .compressed_tensors_wNa16 import WNA16_SUPPORTED_BITS # noqa: F401
11 +from .compressed_tensors_wNa16 import CompressedTensorsWNA16 # noqa: F401
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+1
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
11 from vllm.model_executor.utils import set_weight_attrs
12
13 __all__ = ["CompressedTensorsW4A16Sparse24"]
14 +W4A16SPARSE24_SUPPORTED_BITS = [4]
15
16
17 class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py renamed
+3 -2
@@ -11,10 +11,11 @@ from vllm.model_executor.layers.quantization.gptq_marlin import (
11 marlin_permute_scales)
12 from vllm.model_executor.utils import set_weight_attrs
13
14 -__all__ = ["CompressedTensorsW4A16"]
14 +__all__ = ["CompressedTensorsWNA16"]
15 +WNA16_SUPPORTED_BITS = [4, 8]
16
17
17 -class CompressedTensorsW4A16(CompressedTensorsScheme):
18 +class CompressedTensorsWNA16(CompressedTensorsScheme):
19
20 def __init__(self,
21 strategy: str,