A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[ Misc ] Clean Up `CompressedTensorsW8A8` (#6113)
Robert Shaw committed
Jul 3, 2024 at 18:50 UTC
62963d129e84d0a0904ee62dbab067a29216e7bf
6 files changed
+44
-95
tests/quantization/test_compressed_tensors.py
+5
-4
@@ -9,8 +9,7 @@ import torch
9
from vllm import SamplingParams
10
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
11
CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
12
- CompressedTensorsW8A8DynamicToken, CompressedTensorsW8A8StaticTensor,
13
- CompressedTensorsWNA16)
12
+ CompressedTensorsW8A8, CompressedTensorsWNA16)
13
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
14
QuantizationType)
15
@@ -38,9 +37,10 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
37
CompressedTensorsLinearMethod)
38
assert isinstance(down_proj.quant_method,
39
CompressedTensorsLinearMethod)
41
- assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8StaticTensor)
40
+ assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8)
41
42
assert qkv_proj.scheme.strategy == strategy
43
+ assert qkv_proj.scheme.is_static_input_scheme
44
expected_type = (torch.int8 if quant_type == QuantizationType.INT else
45
torch.float8_e4m3fn)
46
@@ -79,7 +79,8 @@ def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner, model_args):
79
qkv_proj = layer.self_attn.qkv_proj
80
81
assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
82
- assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8DynamicToken)
82
+ assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8)
83
+ assert not qkv_proj.scheme.is_static_input_scheme
84
assert qkv_proj.scheme.strategy == strategy
85
assert qkv_proj.weight.dtype is torch.int8
86
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+5
-6
@@ -9,8 +9,7 @@ from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
9
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10
W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
11
CompressedTensorsScheme, CompressedTensorsW4A16Sparse24,
12
- CompressedTensorsW8A8DynamicToken, CompressedTensorsW8A8StaticTensor,
13
- CompressedTensorsWNA16)
12
+ CompressedTensorsW8A8, CompressedTensorsWNA16)
13
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
14
CompressionFormat, QuantizationArgs, QuantizationStrategy,
15
find_first_name_or_class_match)
@@ -150,12 +149,12 @@ class CompressedTensorsConfig(QuantizationConfig):
149
150
if self.quant_format == CompressionFormat.int_quantized.value:
151
if self._is_static_tensor_w8a8(weight_quant, input_quant):
153
- return CompressedTensorsW8A8StaticTensor(
154
- strategy=weight_quant.strategy)
152
+ return CompressedTensorsW8A8(strategy=weight_quant.strategy,
153
+ is_static_input_scheme=True)
154
155
if self._is_dynamic_token_w8a8(weight_quant, input_quant):
157
- return CompressedTensorsW8A8DynamicToken(
158
- strategy=weight_quant.strategy)
156
+ return CompressedTensorsW8A8(strategy=weight_quant.strategy,
157
+ is_static_input_scheme=False)
158
159
raise NotImplementedError(
160
"No compressed-tensors compatible scheme was found.")
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
+1
-4
@@ -3,9 +3,6 @@ from .compressed_tensors_unquantized import ( # noqa: F401
3
CompressedTensorsUnquantized)
4
from .compressed_tensors_w4a16_24 import ( # noqa: F401
5
W4A16SPARSE24_SUPPORTED_BITS, CompressedTensorsW4A16Sparse24)
6
-from .compressed_tensors_w8a8_dynamictoken import ( # noqa: F401, E501
7
- CompressedTensorsW8A8DynamicToken)
8
-from .compressed_tensors_w8a8_statictensor import ( # noqa: F401, E501
9
- CompressedTensorsW8A8StaticTensor)
6
+from .compressed_tensors_w8a8 import CompressedTensorsW8A8 # noqa: F401
7
from .compressed_tensors_wNa16 import WNA16_SUPPORTED_BITS # noqa: F401
8
from .compressed_tensors_wNa16 import CompressedTensorsWNA16 # noqa: F401
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8.py
+33
-1
@@ -3,6 +3,7 @@ from typing import Callable, List, Tuple, Union
3
import torch
4
from torch.nn import Parameter
5
6
+from vllm import _custom_ops as ops
7
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
CompressedTensorsScheme)
9
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
@@ -12,8 +13,9 @@ from vllm.model_executor.utils import set_weight_attrs
13
14
class CompressedTensorsW8A8(CompressedTensorsScheme):
15
15
- def __init__(self, strategy: str):
16
+ def __init__(self, strategy: str, is_static_input_scheme: bool):
17
self.strategy = strategy
18
+ self.is_static_input_scheme = is_static_input_scheme
19
20
# Cutlass kernels support only per-tensor and per-channel cases.
21
# So if we have a fused module (QKV, MLP) with per tensor scales (thus N
@@ -36,6 +38,10 @@ class CompressedTensorsW8A8(CompressedTensorsScheme):
38
layer.weight_scale = Parameter(weight_scale_channel,
39
requires_grad=False)
40
41
+ # transpose weights for cutlass.
42
+ weight = layer.weight
43
+ layer.weight = Parameter(weight.t(), requires_grad=False)
44
+
45
def create_weights(self, layer: torch.nn.Module,
46
output_partition_sizes: List[int],
47
input_size_per_partition: int,
@@ -75,3 +81,29 @@ class CompressedTensorsW8A8(CompressedTensorsScheme):
81
"output_dim": 0,
82
"weight_loader": weight_loader,
83
})
84
+
85
+ # INPUT SCALE
86
+ # Static quantization: load from disk.
87
+ if self.is_static_input_scheme:
88
+ input_scale = Parameter(torch.empty(1, dtype=torch.float32),
89
+ requires_grad=False)
90
+ layer.register_parameter("input_scale", input_scale)
91
+ set_weight_attrs(input_scale, {
92
+ "weight_loader": weight_loader,
93
+ "ignore_warning": True,
94
+ })
95
+ # Dynamic quantization: set to None.
96
+ else:
97
+ layer.input_scale = None
98
+
99
+ def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
100
+ # ops.scaled_int8_quant supports both dynamic and static quant.
101
+ # * dynamic, layer.input_scale is None and x_scale computed from x.
102
+ # * static, layer.input_scale is scalar and x_scale is input_scale.
103
+ x_q, x_scale = ops.scaled_int8_quant(x, layer.input_scale)
104
+
105
+ return ops.cutlass_scaled_mm(x_q,
106
+ layer.weight,
107
+ scale_a=x_scale,
108
+ scale_b=layer.weight_scale,
109
+ out_dtype=x.dtype)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_dynamictoken.py
deleted
-33
@@ -1,33 +0,0 @@
1
-from typing import Callable, List
2
-
3
-import torch
4
-
5
-from vllm import _custom_ops as custom_ops
6
-from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_w8a8 import ( # noqa: E501
7
- CompressedTensorsW8A8)
8
-
9
-__all__ = ["CompressedTensorsW8A8DynamicToken"]
10
-
11
-
12
-class CompressedTensorsW8A8DynamicToken(CompressedTensorsW8A8):
13
-
14
- def create_weights(self, layer: torch.nn.Module,
15
- output_partition_sizes: List[int],
16
- input_size_per_partition: int,
17
- params_dtype: torch.dtype, weight_loader: Callable,
18
- **kwargs):
19
-
20
- super().create_weights(
21
- layer=layer,
22
- output_partition_sizes=output_partition_sizes,
23
- input_size_per_partition=input_size_per_partition,
24
- params_dtype=params_dtype,
25
- weight_loader=weight_loader)
26
-
27
- def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
28
- weight = layer.weight
29
- weight_scale = layer.weight_scale
30
-
31
- x_q, input_scales = custom_ops.scaled_int8_quant(x)
32
- return custom_ops.cutlass_scaled_mm(x_q, weight.t(), input_scales,
33
- weight_scale, x.dtype)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_statictensor.py
deleted
-47
@@ -1,47 +0,0 @@
1
-from typing import Callable, List
2
-
3
-import torch
4
-from torch.nn import Parameter
5
-
6
-from vllm import _custom_ops as custom_ops
7
-from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_w8a8 import ( # noqa: E501
8
- CompressedTensorsW8A8)
9
-from vllm.model_executor.utils import set_weight_attrs
10
-
11
-__all__ = ["CompressedTensorsW8A8StaticTensor"]
12
-
13
-
14
-class CompressedTensorsW8A8StaticTensor(CompressedTensorsW8A8):
15
-
16
- def create_weights(self, layer: torch.nn.Module,
17
- output_partition_sizes: List[int],
18
- input_size_per_partition: int,
19
- params_dtype: torch.dtype, weight_loader: Callable,
20
- **kwargs):
21
-
22
- super().create_weights(
23
- layer=layer,
24
- output_partition_sizes=output_partition_sizes,
25
- input_size_per_partition=input_size_per_partition,
26
- params_dtype=params_dtype,
27
- weight_loader=weight_loader)
28
-
29
- input_scale = Parameter(torch.empty(1, dtype=torch.float32),
30
- requires_grad=False)
31
-
32
- layer.register_parameter("input_scale", input_scale)
33
- set_weight_attrs(input_scale, {
34
- "weight_loader": weight_loader,
35
- "ignore_warning": True,
36
- })
37
-
38
- def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
39
- weight = layer.weight
40
- weight_scale = layer.weight_scale
41
- act_scale = layer.input_scale
42
-
43
- # Input quantize
44
- x_q, _ = custom_ops.scaled_int8_quant(x, act_scale)
45
-
46
- return custom_ops.cutlass_scaled_mm(x_q, weight.t(), act_scale,
47
- weight_scale, x.dtype)