A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Misc] Add channel-wise quantization support for w8a8 dynamic per token activation quantization (#5542)
Dipika Sikka committed
Jun 18, 2024 at 12:45 UTC
95db455e7f337e99ffafd0b14367a7cbc11dca43
4 files changed
+45
-32
tests/quantization/test_compressed_tensors.py
+9
-4
@@ -14,7 +14,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
14
15
16
def test_compressed_tensors_w8a8_static_setup(vllm_runner):
17
- model_path = "nm-testing/tinyllama-oneshot-w8a8-static-v2"
17
+ model_path = "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change"
18
with vllm_runner(model_path, enforce_eager=True) as llm:
19
model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
20
layer = model.model.layers[0]
@@ -43,15 +43,19 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner):
43
44
45
def test_compressed_tensors_no_enforce_eager(vllm_runner):
46
- model_path = "nm-testing/tinyllama-oneshot-w8a8-static-v2"
46
+ model_path = "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change"
47
with vllm_runner(model_path) as llm:
48
sampling_params = SamplingParams()
49
output = llm.generate("Hello world!", sampling_params=sampling_params)
50
assert output
51
52
53
-def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner):
54
- model_path = "nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2"
53
+@pytest.mark.parametrize("model_args", [
54
+ ("nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2", "tensor"),
55
+ ("nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2", "channel"),
56
+])
57
+def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner, model_args):
58
+ model_path, strategy = model_args
59
with vllm_runner(model_path, dtype=torch.float16) as llm:
60
model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
61
layer = model.model.layers[0]
@@ -60,6 +64,7 @@ def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner):
64
65
assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
66
assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8DynamicToken)
67
+ assert qkv_proj.scheme.strategy == strategy
68
assert qkv_proj.weight.dtype is torch.int8
69
70
vllm/model_executor/layers/linear.py
-13
@@ -468,13 +468,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
468
"MergedColumnParallelLinear, assume the weight is "
469
"the same for all partitions.")
470
471
- if fp8_scales_shard_indexer is None:
472
- if len(param_data.shape) == 0:
473
- param_data = param_data.reshape(1)
474
-
475
- if len(loaded_weight.shape) == 0:
476
- loaded_weight = loaded_weight.reshape(1)
477
-
471
assert param_data.shape == loaded_weight.shape
472
param_data.copy_(loaded_weight)
473
@@ -686,12 +679,6 @@ class QKVParallelLinear(ColumnParallelLinear):
679
"QKVParallelLinear, assume the weight is the same "
680
"for all partitions.")
681
689
- if len(param_data.shape) == 0:
690
- param_data = param_data.reshape(1)
691
-
692
- if len(loaded_weight.shape) == 0:
693
- loaded_weight = loaded_weight.reshape(1)
694
-
682
assert param_data.shape == loaded_weight.shape
683
param_data.copy_(loaded_weight)
684
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+8
-6
@@ -95,14 +95,15 @@ class CompressedTensorsConfig(QuantizationConfig):
95
def _is_dynamic_token_w8a8(self, weight_quant: BaseModel,
96
input_quant: BaseModel) -> bool:
97
is_8_bits = weight_quant.num_bits == input_quant.num_bits == 8
98
- is_token_tensor = (weight_quant.strategy
99
- == QuantizationStrategy.TENSOR.value) and (
100
- input_quant.strategy
101
- == QuantizationStrategy.TOKEN.value)
98
+ weight_strategy = (
99
+ weight_quant.strategy == QuantizationStrategy.TENSOR.value
100
+ or weight_quant.strategy == QuantizationStrategy.CHANNEL.value)
101
+ is_token = (weight_strategy and input_quant.strategy
102
+ == QuantizationStrategy.TOKEN.value)
103
is_symmetric = weight_quant.symmetric and input_quant.symmetric
104
is_dynamic = not weight_quant.dynamic and input_quant.dynamic
105
105
- return is_8_bits and is_token_tensor and is_symmetric and is_dynamic
106
+ return is_8_bits and is_token and is_symmetric and is_dynamic
107
108
def _is_w4a16(self, weight_quant: BaseModel,
109
input_quant: BaseModel) -> bool:
@@ -133,7 +134,8 @@ class CompressedTensorsConfig(QuantizationConfig):
134
return CompressedTensorsW8A8StaticTensor()
135
136
if self._is_dynamic_token_w8a8(weight_quant, input_quant):
136
- return CompressedTensorsW8A8DynamicToken()
137
+ return CompressedTensorsW8A8DynamicToken(
138
+ strategy=weight_quant.strategy)
139
140
raise NotImplementedError(
141
"No compressed-tensors compatible scheme was found.")
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_dynamictoken.py
+28
-9
@@ -6,6 +6,8 @@ from torch.nn import Parameter
6
from vllm import _custom_ops as custom_ops
7
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
CompressedTensorsScheme)
9
+from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
10
+ QuantizationStrategy)
11
from vllm.model_executor.utils import set_weight_attrs
12
13
__all__ = ["CompressedTensorsW8A8DynamicToken"]
@@ -13,6 +15,9 @@ __all__ = ["CompressedTensorsW8A8DynamicToken"]
15
16
class CompressedTensorsW8A8DynamicToken(CompressedTensorsScheme):
17
18
+ def __init__(self, strategy: str):
19
+ self.strategy = strategy
20
+
21
def _shard_id_as_int(self, shard_id: Union[str, int]) -> int:
22
if isinstance(shard_id, int):
23
return shard_id
@@ -45,11 +50,17 @@ class CompressedTensorsW8A8DynamicToken(CompressedTensorsScheme):
50
# CompressedTensorsW8A8StaticTensor::create_weights for further
51
# information.
52
is_tensor_partitioned = len(output_partition_sizes) != 1
48
- weight_scale_dim = sum(
49
- output_partition_sizes) if is_tensor_partitioned else 1
53
+ # when doing channel-wise quantization, number of scales
54
+ # is equal to output_dim
55
+ weight_scale_dim = sum(output_partition_sizes) if (
56
+ is_tensor_partitioned
57
+ or self.strategy == QuantizationStrategy.CHANNEL) else 1
58
+
59
+ shape: Union[Tuple[int], Tuple[int, int]] = (weight_scale_dim, )
60
+ if self.strategy == QuantizationStrategy.CHANNEL:
61
+ shape = (weight_scale_dim, 1)
62
51
- weight_scale = Parameter(torch.empty(weight_scale_dim,
52
- dtype=torch.float32),
63
+ weight_scale = Parameter(torch.empty(*shape, dtype=torch.float32),
64
requires_grad=False)
65
66
weight = Parameter(torch.empty(sum(output_partition_sizes),
@@ -67,12 +78,20 @@ class CompressedTensorsW8A8DynamicToken(CompressedTensorsScheme):
78
})
79
80
layer.register_parameter("weight_scale", weight_scale)
70
- set_weight_attrs(
71
- weight_scale, {
72
- "weight_loader": weight_loader,
73
- "shard_splitter": self.scales_shard_splitter,
74
- "logical_widths": output_partition_sizes
81
+ set_weight_attrs(weight_scale, {"weight_loader": weight_loader})
82
+
83
+ # Don't need a shard_splitter for channel-wise quantization
84
+ # Use the default loading method
85
+ if self.strategy == QuantizationStrategy.CHANNEL:
86
+ set_weight_attrs(weight_scale, {
87
+ "output_dim": 0,
88
})
89
+ else:
90
+ set_weight_attrs(
91
+ weight_scale, {
92
+ "logical_widths": output_partition_sizes,
93
+ "shard_splitter": self.scales_shard_splitter,
94
+ })
95
96
def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
97
weight = layer.weight