@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4a30d7e3cc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Misc] Add per channel support for static activation quantization; update w8a8 schemes to share base classes (#5650)

Dipika Sikka committed Jun 19, 2024 at 18:06 UTC 4a30d7e3ccae6e977d728e2157aaa11ac0fed549
5 files changed +121 -136
tests/quantization/test_compressed_tensors.py
+10 -4
@@ -13,8 +13,12 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
13 CompressedTensorsW8A8StaticTensor)
14
15
16 -def test_compressed_tensors_w8a8_static_setup(vllm_runner):
17 - model_path = "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change"
16 +@pytest.mark.parametrize("model_args", [
17 + ("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", "tensor"),
18 + ("nm-testing/tinyllama-oneshot-w8-channel-a8-tensor", "channel"),
19 +])
20 +def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
21 + model_path, strategy = model_args
22 with vllm_runner(model_path, enforce_eager=True) as llm:
23 model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
24 layer = model.model.layers[0]
@@ -33,12 +37,14 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner):
37
38 assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8StaticTensor)
39
40 + assert qkv_proj.scheme.strategy == strategy
41 assert qkv_proj.weight.dtype is torch.int8
42 assert o_proj.weight.dtype is torch.int8
43 assert gate_up_proj.weight.dtype is torch.int8
44
40 - assert qkv_proj.weight_scale.shard_splitter is not None
41 - assert qkv_proj.weight_scale.logical_widths is not None
45 + if qkv_proj.scheme.strategy == "tensor":
46 + assert qkv_proj.weight_scale.shard_splitter is not None
47 + assert qkv_proj.weight_scale.logical_widths is not None
48 assert qkv_proj.input_scale.dtype is torch.float32
49
50
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+7 -3
@@ -85,8 +85,11 @@ class CompressedTensorsConfig(QuantizationConfig):
85 def _is_static_tensor_w8a8(self, weight_quant: BaseModel,
86 input_quant: BaseModel) -> bool:
87 is_8_bits = weight_quant.num_bits == input_quant.num_bits == 8
88 - is_tensor = (weight_quant.strategy == input_quant.strategy ==
89 - QuantizationStrategy.TENSOR.value)
88 + weight_strategy = (
89 + weight_quant.strategy == QuantizationStrategy.TENSOR.value
90 + or weight_quant.strategy == QuantizationStrategy.CHANNEL.value)
91 + is_tensor = (weight_strategy and input_quant.strategy
92 + == QuantizationStrategy.TENSOR.value)
93 is_symmetric = weight_quant.symmetric and input_quant.symmetric
94 is_static = not weight_quant.dynamic and not input_quant.dynamic
95
@@ -131,7 +134,8 @@ class CompressedTensorsConfig(QuantizationConfig):
134
135 if self.quant_format == CompressionFormat.int_quantized.value:
136 if self._is_static_tensor_w8a8(weight_quant, input_quant):
134 - return CompressedTensorsW8A8StaticTensor()
137 + return CompressedTensorsW8A8StaticTensor(
138 + strategy=weight_quant.strategy)
139
140 if self._is_dynamic_token_w8a8(weight_quant, input_quant):
141 return CompressedTensorsW8A8DynamicToken(
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8.py new
+84
@@ -0,0 +1,84 @@
1 +from typing import Callable, List, Tuple, Union
2 +
3 +import torch
4 +from torch.nn import Parameter
5 +
6 +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7 + CompressedTensorsScheme)
8 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9 + QuantizationStrategy)
10 +from vllm.model_executor.utils import set_weight_attrs
11 +
12 +
13 +class CompressedTensorsW8A8(CompressedTensorsScheme):
14 +
15 + def __init__(self, strategy: str):
16 + self.strategy = strategy
17 +
18 + def _shard_id_as_int(self, shard_id: Union[str, int]) -> int:
19 + if isinstance(shard_id, int):
20 + return shard_id
21 +
22 + assert isinstance(shard_id, str)
23 + qkv_idxs = {"q": 0, "k": 1, "v": 2}
24 + assert shard_id in qkv_idxs
25 + return qkv_idxs[shard_id]
26 +
27 + def scales_shard_splitter(
28 + self, param: torch.Tensor, loaded_weight: torch.Tensor,
29 + shard_id: Union[str, int],
30 + logical_widths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
31 + shard_id = self._shard_id_as_int(shard_id)
32 + offset = sum(logical_widths[:shard_id])
33 + size = logical_widths[shard_id]
34 + # update loaded weight with copies for broadcast.
35 + loaded_weight = loaded_weight.repeat(size)
36 + return param[offset:offset + size], loaded_weight
37 +
38 + def create_weights(self, layer: torch.nn.Module,
39 + output_partition_sizes: List[int],
40 + input_size_per_partition: int,
41 + params_dtype: torch.dtype, weight_loader: Callable,
42 + **kwargs):
43 +
44 + is_tensor_partitioned = len(output_partition_sizes) != 1
45 + weight_scale_dim = sum(output_partition_sizes) if (
46 + is_tensor_partitioned
47 + or self.strategy == QuantizationStrategy.CHANNEL) else 1
48 +
49 + shape: Union[Tuple[int], Tuple[int, int]] = (weight_scale_dim, )
50 + if self.strategy == QuantizationStrategy.CHANNEL:
51 + shape = (weight_scale_dim, 1)
52 +
53 + weight_scale = Parameter(torch.empty(*shape, dtype=torch.float32),
54 + requires_grad=False)
55 +
56 + layer.register_parameter("weight_scale", weight_scale)
57 + set_weight_attrs(weight_scale, {"weight_loader": weight_loader})
58 +
59 + weight = Parameter(torch.empty(sum(output_partition_sizes),
60 + input_size_per_partition,
61 + dtype=torch.int8),
62 + requires_grad=False)
63 +
64 + layer.register_parameter("weight", weight)
65 + set_weight_attrs(
66 + weight, {
67 + "input_dim": 1,
68 + "output_dim": 0,
69 + "weight_loader": weight_loader,
70 + "logical_widths": output_partition_sizes
71 + })
72 +
73 + # Don't need a shard_splitter for channel-wise quantization
74 + # Use the default loading method
75 + if self.strategy == QuantizationStrategy.CHANNEL:
76 + set_weight_attrs(weight_scale, {
77 + "output_dim": 0,
78 + })
79 + else:
80 + set_weight_attrs(
81 + weight_scale, {
82 + "logical_widths": output_partition_sizes,
83 + "shard_splitter": self.scales_shard_splitter,
84 + })
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_dynamictoken.py
+10 -79
@@ -1,42 +1,15 @@
1 -from typing import Callable, List, Tuple, Union
1 +from typing import Callable, List
2
3 import torch
4 -from torch.nn import Parameter
4
5 from vllm import _custom_ops as custom_ops
7 -from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 - CompressedTensorsScheme)
9 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
10 - QuantizationStrategy)
11 -from vllm.model_executor.utils import set_weight_attrs
6 +from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_w8a8 import ( # noqa: E501
7 + CompressedTensorsW8A8)
8
9 __all__ = ["CompressedTensorsW8A8DynamicToken"]
10
11
16 -class CompressedTensorsW8A8DynamicToken(CompressedTensorsScheme):
17 -
18 - def __init__(self, strategy: str):
19 - self.strategy = strategy
20 -
21 - def _shard_id_as_int(self, shard_id: Union[str, int]) -> int:
22 - if isinstance(shard_id, int):
23 - return shard_id
24 -
25 - assert isinstance(shard_id, str)
26 - qkv_idxs = {"q": 0, "k": 1, "v": 2}
27 - assert shard_id in qkv_idxs
28 - return qkv_idxs[shard_id]
29 -
30 - def scales_shard_splitter(
31 - self, param: torch.Tensor, loaded_weight: torch.Tensor,
32 - shard_id: Union[str, int],
33 - logical_widths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
34 - shard_id = self._shard_id_as_int(shard_id)
35 - offset = sum(logical_widths[:shard_id])
36 - size = logical_widths[shard_id]
37 - # update loaded weight with copies for broadcast.
38 - loaded_weight = loaded_weight.repeat(size)
39 - return param[offset:offset + size], loaded_weight
12 +class CompressedTensorsW8A8DynamicToken(CompressedTensorsW8A8):
13
14 def create_weights(self, layer: torch.nn.Module,
15 output_partition_sizes: List[int],
@@ -44,54 +17,12 @@ class CompressedTensorsW8A8DynamicToken(CompressedTensorsScheme):
17 params_dtype: torch.dtype, weight_loader: Callable,
18 **kwargs):
19
47 - # When the scales have a single value, it is required that they be
48 - # on the CPU for performance and CUDA Graphs compatibility. Please
49 - # refer to the comment in
50 - # CompressedTensorsW8A8StaticTensor::create_weights for further
51 - # information.
52 - is_tensor_partitioned = len(output_partition_sizes) != 1
53 - # when doing channel-wise quantization, number of scales
54 - # is equal to output_dim
55 - weight_scale_dim = sum(output_partition_sizes) if (
56 - is_tensor_partitioned
57 - or self.strategy == QuantizationStrategy.CHANNEL) else 1
58 -
59 - shape: Union[Tuple[int], Tuple[int, int]] = (weight_scale_dim, )
60 - if self.strategy == QuantizationStrategy.CHANNEL:
61 - shape = (weight_scale_dim, 1)
62 -
63 - weight_scale = Parameter(torch.empty(*shape, dtype=torch.float32),
64 - requires_grad=False)
65 -
66 - weight = Parameter(torch.empty(sum(output_partition_sizes),
67 - input_size_per_partition,
68 - dtype=torch.int8),
69 - requires_grad=False)
70 -
71 - layer.register_parameter("weight", weight)
72 - set_weight_attrs(
73 - weight, {
74 - "input_dim": 1,
75 - "output_dim": 0,
76 - "weight_loader": weight_loader,
77 - "logical_widths": output_partition_sizes
78 - })
79 -
80 - layer.register_parameter("weight_scale", weight_scale)
81 - set_weight_attrs(weight_scale, {"weight_loader": weight_loader})
82 -
83 - # Don't need a shard_splitter for channel-wise quantization
84 - # Use the default loading method
85 - if self.strategy == QuantizationStrategy.CHANNEL:
86 - set_weight_attrs(weight_scale, {
87 - "output_dim": 0,
88 - })
89 - else:
90 - set_weight_attrs(
91 - weight_scale, {
92 - "logical_widths": output_partition_sizes,
93 - "shard_splitter": self.scales_shard_splitter,
94 - })
20 + super().create_weights(
21 + layer=layer,
22 + output_partition_sizes=output_partition_sizes,
23 + input_size_per_partition=input_size_per_partition,
24 + params_dtype=params_dtype,
25 + weight_loader=weight_loader)
26
27 def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
28 weight = layer.weight
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_statictensor.py
+10 -50
@@ -1,37 +1,17 @@
1 -from typing import Callable, List, Tuple, Union
1 +from typing import Callable, List
2
3 import torch
4 from torch.nn import Parameter
5
6 from vllm import _custom_ops as custom_ops
7 -from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 - CompressedTensorsScheme)
7 +from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_w8a8 import ( # noqa: E501
8 + CompressedTensorsW8A8)
9 from vllm.model_executor.utils import set_weight_attrs
10
11 __all__ = ["CompressedTensorsW8A8StaticTensor"]
12
13
14 -class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
15 -
16 - def _shard_id_as_int(self, shard_id: Union[str, int]) -> int:
17 - if isinstance(shard_id, int):
18 - return shard_id
19 -
20 - assert isinstance(shard_id, str)
21 - qkv_idxs = {"q": 0, "k": 1, "v": 2}
22 - assert shard_id in qkv_idxs
23 - return qkv_idxs[shard_id]
24 -
25 - def scales_shard_splitter(
26 - self, param: torch.Tensor, loaded_weight: torch.Tensor,
27 - shard_id: Union[str, int],
28 - logical_widths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
29 - shard_id = self._shard_id_as_int(shard_id)
30 - offset = sum(logical_widths[:shard_id])
31 - size = logical_widths[shard_id]
32 - # update loaded weight with copies for broadcast.
33 - loaded_weight = loaded_weight.repeat(size)
34 - return param[offset:offset + size], loaded_weight
14 +class CompressedTensorsW8A8StaticTensor(CompressedTensorsW8A8):
15
16 def create_weights(self, layer: torch.nn.Module,
17 output_partition_sizes: List[int],
@@ -39,41 +19,21 @@ class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
19 params_dtype: torch.dtype, weight_loader: Callable,
20 **kwargs):
21
42 - is_tensor_partitioned = len(output_partition_sizes) != 1
43 - weight_scale_dim = sum(
44 - output_partition_sizes) if is_tensor_partitioned else 1
22 + super().create_weights(
23 + layer=layer,
24 + output_partition_sizes=output_partition_sizes,
25 + input_size_per_partition=input_size_per_partition,
26 + params_dtype=params_dtype,
27 + weight_loader=weight_loader)
28
29 input_scale = Parameter(torch.empty(1, dtype=torch.float32),
30 requires_grad=False)
31
49 - weight_scale = Parameter(torch.empty(weight_scale_dim,
50 - dtype=torch.float32),
51 - requires_grad=False)
52 -
53 - weight = Parameter(torch.empty(sum(output_partition_sizes),
54 - input_size_per_partition,
55 - dtype=torch.int8),
56 - requires_grad=False)
57 -
58 - layer.register_parameter("weight", weight)
59 - set_weight_attrs(weight, {
60 - "weight_loader": weight_loader,
61 - "input_dim": 1,
62 - "output_dim": 0,
63 - })
32 layer.register_parameter("input_scale", input_scale)
33 set_weight_attrs(input_scale, {
34 "weight_loader": weight_loader,
35 "ignore_warning": True,
36 })
69 - layer.register_parameter("weight_scale", weight_scale)
70 - set_weight_attrs(
71 - weight_scale, {
72 - "weight_loader": weight_loader,
73 - "shard_splitter": self.scales_shard_splitter,
74 - "logical_widths": output_partition_sizes,
75 - "ignore_warning": True,
76 - })
37
38 def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
39 weight = layer.weight