@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 62963d129e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[ Misc ] Clean Up `CompressedTensorsW8A8` (#6113)

Robert Shaw committed Jul 3, 2024 at 18:50 UTC 62963d129e84d0a0904ee62dbab067a29216e7bf
6 files changed +44 -95
tests/quantization/test_compressed_tensors.py
+5 -4
@@ -9,8 +9,7 @@ import torch
9 from vllm import SamplingParams
10 from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
11 CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
12 - CompressedTensorsW8A8DynamicToken, CompressedTensorsW8A8StaticTensor,
13 - CompressedTensorsWNA16)
12 + CompressedTensorsW8A8, CompressedTensorsWNA16)
13 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
14 QuantizationType)
15
@@ -38,9 +37,10 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
37 CompressedTensorsLinearMethod)
38 assert isinstance(down_proj.quant_method,
39 CompressedTensorsLinearMethod)
41 - assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8StaticTensor)
40 + assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8)
41
42 assert qkv_proj.scheme.strategy == strategy
43 + assert qkv_proj.scheme.is_static_input_scheme
44 expected_type = (torch.int8 if quant_type == QuantizationType.INT else
45 torch.float8_e4m3fn)
46
@@ -79,7 +79,8 @@ def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner, model_args):
79 qkv_proj = layer.self_attn.qkv_proj
80
81 assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
82 - assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8DynamicToken)
82 + assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8)
83 + assert not qkv_proj.scheme.is_static_input_scheme
84 assert qkv_proj.scheme.strategy == strategy
85 assert qkv_proj.weight.dtype is torch.int8
86
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+5 -6
@@ -9,8 +9,7 @@ from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
9 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10 W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
11 CompressedTensorsScheme, CompressedTensorsW4A16Sparse24,
12 - CompressedTensorsW8A8DynamicToken, CompressedTensorsW8A8StaticTensor,
13 - CompressedTensorsWNA16)
12 + CompressedTensorsW8A8, CompressedTensorsWNA16)
13 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
14 CompressionFormat, QuantizationArgs, QuantizationStrategy,
15 find_first_name_or_class_match)
@@ -150,12 +149,12 @@ class CompressedTensorsConfig(QuantizationConfig):
149
150 if self.quant_format == CompressionFormat.int_quantized.value:
151 if self._is_static_tensor_w8a8(weight_quant, input_quant):
153 - return CompressedTensorsW8A8StaticTensor(
154 - strategy=weight_quant.strategy)
152 + return CompressedTensorsW8A8(strategy=weight_quant.strategy,
153 + is_static_input_scheme=True)
154
155 if self._is_dynamic_token_w8a8(weight_quant, input_quant):
157 - return CompressedTensorsW8A8DynamicToken(
158 - strategy=weight_quant.strategy)
156 + return CompressedTensorsW8A8(strategy=weight_quant.strategy,
157 + is_static_input_scheme=False)
158
159 raise NotImplementedError(
160 "No compressed-tensors compatible scheme was found.")
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
+1 -4
@@ -3,9 +3,6 @@ from .compressed_tensors_unquantized import ( # noqa: F401
3 CompressedTensorsUnquantized)
4 from .compressed_tensors_w4a16_24 import ( # noqa: F401
5 W4A16SPARSE24_SUPPORTED_BITS, CompressedTensorsW4A16Sparse24)
6 -from .compressed_tensors_w8a8_dynamictoken import ( # noqa: F401, E501
7 - CompressedTensorsW8A8DynamicToken)
8 -from .compressed_tensors_w8a8_statictensor import ( # noqa: F401, E501
9 - CompressedTensorsW8A8StaticTensor)
6 +from .compressed_tensors_w8a8 import CompressedTensorsW8A8 # noqa: F401
7 from .compressed_tensors_wNa16 import WNA16_SUPPORTED_BITS # noqa: F401
8 from .compressed_tensors_wNa16 import CompressedTensorsWNA16 # noqa: F401
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8.py
+33 -1
@@ -3,6 +3,7 @@ from typing import Callable, List, Tuple, Union
3 import torch
4 from torch.nn import Parameter
5
6 +from vllm import _custom_ops as ops
7 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 CompressedTensorsScheme)
9 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
@@ -12,8 +13,9 @@ from vllm.model_executor.utils import set_weight_attrs
13
14 class CompressedTensorsW8A8(CompressedTensorsScheme):
15
15 - def __init__(self, strategy: str):
16 + def __init__(self, strategy: str, is_static_input_scheme: bool):
17 self.strategy = strategy
18 + self.is_static_input_scheme = is_static_input_scheme
19
20 # Cutlass kernels support only per-tensor and per-channel cases.
21 # So if we have a fused module (QKV, MLP) with per tensor scales (thus N
@@ -36,6 +38,10 @@ class CompressedTensorsW8A8(CompressedTensorsScheme):
38 layer.weight_scale = Parameter(weight_scale_channel,
39 requires_grad=False)
40
41 + # transpose weights for cutlass.
42 + weight = layer.weight
43 + layer.weight = Parameter(weight.t(), requires_grad=False)
44 +
45 def create_weights(self, layer: torch.nn.Module,
46 output_partition_sizes: List[int],
47 input_size_per_partition: int,
@@ -75,3 +81,29 @@ class CompressedTensorsW8A8(CompressedTensorsScheme):
81 "output_dim": 0,
82 "weight_loader": weight_loader,
83 })
84 +
85 + # INPUT SCALE
86 + # Static quantization: load from disk.
87 + if self.is_static_input_scheme:
88 + input_scale = Parameter(torch.empty(1, dtype=torch.float32),
89 + requires_grad=False)
90 + layer.register_parameter("input_scale", input_scale)
91 + set_weight_attrs(input_scale, {
92 + "weight_loader": weight_loader,
93 + "ignore_warning": True,
94 + })
95 + # Dynamic quantization: set to None.
96 + else:
97 + layer.input_scale = None
98 +
99 + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
100 + # ops.scaled_int8_quant supports both dynamic and static quant.
101 + # * dynamic, layer.input_scale is None and x_scale computed from x.
102 + # * static, layer.input_scale is scalar and x_scale is input_scale.
103 + x_q, x_scale = ops.scaled_int8_quant(x, layer.input_scale)
104 +
105 + return ops.cutlass_scaled_mm(x_q,
106 + layer.weight,
107 + scale_a=x_scale,
108 + scale_b=layer.weight_scale,
109 + out_dtype=x.dtype)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_dynamictoken.py deleted
-33
@@ -1,33 +0,0 @@
1 -from typing import Callable, List
2 -
3 -import torch
4 -
5 -from vllm import _custom_ops as custom_ops
6 -from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_w8a8 import ( # noqa: E501
7 - CompressedTensorsW8A8)
8 -
9 -__all__ = ["CompressedTensorsW8A8DynamicToken"]
10 -
11 -
12 -class CompressedTensorsW8A8DynamicToken(CompressedTensorsW8A8):
13 -
14 - def create_weights(self, layer: torch.nn.Module,
15 - output_partition_sizes: List[int],
16 - input_size_per_partition: int,
17 - params_dtype: torch.dtype, weight_loader: Callable,
18 - **kwargs):
19 -
20 - super().create_weights(
21 - layer=layer,
22 - output_partition_sizes=output_partition_sizes,
23 - input_size_per_partition=input_size_per_partition,
24 - params_dtype=params_dtype,
25 - weight_loader=weight_loader)
26 -
27 - def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
28 - weight = layer.weight
29 - weight_scale = layer.weight_scale
30 -
31 - x_q, input_scales = custom_ops.scaled_int8_quant(x)
32 - return custom_ops.cutlass_scaled_mm(x_q, weight.t(), input_scales,
33 - weight_scale, x.dtype)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_statictensor.py deleted
-47
@@ -1,47 +0,0 @@
1 -from typing import Callable, List
2 -
3 -import torch
4 -from torch.nn import Parameter
5 -
6 -from vllm import _custom_ops as custom_ops
7 -from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_w8a8 import ( # noqa: E501
8 - CompressedTensorsW8A8)
9 -from vllm.model_executor.utils import set_weight_attrs
10 -
11 -__all__ = ["CompressedTensorsW8A8StaticTensor"]
12 -
13 -
14 -class CompressedTensorsW8A8StaticTensor(CompressedTensorsW8A8):
15 -
16 - def create_weights(self, layer: torch.nn.Module,
17 - output_partition_sizes: List[int],
18 - input_size_per_partition: int,
19 - params_dtype: torch.dtype, weight_loader: Callable,
20 - **kwargs):
21 -
22 - super().create_weights(
23 - layer=layer,
24 - output_partition_sizes=output_partition_sizes,
25 - input_size_per_partition=input_size_per_partition,
26 - params_dtype=params_dtype,
27 - weight_loader=weight_loader)
28 -
29 - input_scale = Parameter(torch.empty(1, dtype=torch.float32),
30 - requires_grad=False)
31 -
32 - layer.register_parameter("input_scale", input_scale)
33 - set_weight_attrs(input_scale, {
34 - "weight_loader": weight_loader,
35 - "ignore_warning": True,
36 - })
37 -
38 - def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
39 - weight = layer.weight
40 - weight_scale = layer.weight_scale
41 - act_scale = layer.input_scale
42 -
43 - # Input quantize
44 - x_q, _ = custom_ops.scaled_int8_quant(x, act_scale)
45 -
46 - return custom_ops.cutlass_scaled_mm(x_q, weight.t(), act_scale,
47 - weight_scale, x.dtype)