A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[ Kernel ] Fp8 Channelwise Weight Support (#6487)
Robert Shaw committed
Jul 17, 2024 at 23:18 UTC
18fecc3559cad615fd8565d1a001557ec355f285
4 files changed
+76
-35
vllm/config.py
+2
-1
@@ -238,7 +238,8 @@ class ModelConfig:
238
f"{self.quantization} quantization is currently not "
239
f"supported in ROCm.")
240
if (self.quantization
241
- not in ("fp8", "marlin", "gptq_marlin_24", "gptq_marlin")):
241
+ not in ("fp8", "marlin", "gptq_marlin_24", "gptq_marlin",
242
+ "compressed_tensors")):
243
logger.warning(
244
"%s quantization is not fully "
245
"optimized yet. The speed can be slower than "
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+11
-7
@@ -13,7 +13,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
13
CompressedTensorsWNA16)
14
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15
CompressionFormat, QuantizationArgs, QuantizationStrategy,
16
- QuantizationType, find_first_name_or_class_match)
16
+ QuantizationType, find_first_name_or_class_match,
17
+ is_activation_quantization_format)
18
from vllm.platforms import current_platform
19
20
@@ -132,10 +133,11 @@ class CompressedTensorsConfig(QuantizationConfig):
133
# Confirm weight scheme is supported.
134
is_symmetric_weight = weight_quant.symmetric
135
is_static_weight = not weight_quant.dynamic
135
- is_per_tensor_weight = (
136
- weight_quant.strategy == QuantizationStrategy.TENSOR)
136
+ is_per_tensor_or_channel_weight = (weight_quant.strategy in [
137
+ QuantizationStrategy.TENSOR, QuantizationStrategy.CHANNEL
138
+ ])
139
if not (is_symmetric_weight and is_static_weight
138
- and is_per_tensor_weight):
140
+ and is_per_tensor_or_channel_weight):
141
return False
142
143
# Dynamic quantization is always supported if weights supported.
@@ -167,6 +169,7 @@ class CompressedTensorsConfig(QuantizationConfig):
169
def _get_schema(self, weight_quant: BaseModel,
170
input_quant: BaseModel) -> "CompressedTensorsScheme":
171
172
+ # Detect If Mixed Precision
173
if self._is_wNa16_group_channel(weight_quant, input_quant):
174
self._check_gptq_and_marlin_can_run()
175
if (self.quant_format == CompressionFormat.marlin_24.value
@@ -182,11 +185,12 @@ class CompressedTensorsConfig(QuantizationConfig):
185
strategy=weight_quant.strategy,
186
group_size=weight_quant.group_size)
187
185
- if (self.quant_format == CompressionFormat.int_quantized.value or
186
- self.quant_format == CompressionFormat.float_quantized.value):
188
+ # Detect If Activation Quantization.
189
+ if is_activation_quantization_format(self.quant_format):
190
if self._is_fp8_w8a8(weight_quant, input_quant):
191
return CompressedTensorsW8A8Fp8(
189
- input_dynamic=input_quant.dynamic)
192
+ strategy=weight_quant.strategy,
193
+ is_static_input_scheme=(not input_quant.dynamic))
194
195
if self._is_static_tensor_w8a8(weight_quant, input_quant):
196
return CompressedTensorsW8A8Int8(
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+53
-27
@@ -1,11 +1,15 @@
1
from typing import Callable, List, Optional
2
3
import torch
4
+from torch.nn import Parameter
5
6
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7
CompressedTensorsScheme)
8
+from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9
+ QuantizationStrategy)
10
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
8
- apply_fp8_linear, create_per_tensor_scale_param, cutlass_fp8_supported,
11
+ apply_fp8_linear, create_per_channel_scale_param,
12
+ create_per_tensor_scale_param, cutlass_fp8_supported,
13
requantize_with_max_scale)
14
from vllm.model_executor.utils import set_weight_attrs
15
@@ -14,39 +18,56 @@ __all__ = ["CompressedTensorsW8A8Fp8"]
18
19
class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
20
17
- def __init__(self, input_dynamic: bool):
18
- self.input_dynamic = input_dynamic
21
+ def __init__(self, strategy: str, is_static_input_scheme: bool):
22
+ self.strategy = strategy
23
+ self.is_static_input_scheme = is_static_input_scheme
24
self.cutlass_fp8_supported = cutlass_fp8_supported()
25
21
- # W8A8-Fp8 kernels support only per-tensor and per-channel cases.
22
- # So if we have a fused module (QKV, MLP) with per tensor scales (thus N
23
- # scales being passed to the kernel), we requantize with a single scale.
26
+ # On Lovelace, fail for now if channelwise.
27
+ # TODO: (@tms) fallback
28
+ if (not self.cutlass_fp8_supported
29
+ and self.strategy == QuantizationStrategy.CHANNEL):
30
+ raise ValueError(
31
+ "Channelwise fp8 quantization requires vLLM's custom "
32
+ "cutlass kernels, which are not supported on your device."
33
+ "Consider quantizing with per tensor scales or upgrading "
34
+ "to Hopper.")
35
+
36
def process_weights_after_loading(self, layer) -> None:
25
- # Dequant -> Quant with max scale.
26
- max_w_scale, weight = requantize_with_max_scale(
27
- weight=layer.weight,
28
- weight_scale=layer.weight_scale,
29
- logical_widths=layer.logical_widths,
30
- )
31
-
32
- # Update layer with new values.
33
- layer.weight = torch.nn.Parameter(weight.t(), requires_grad=False)
34
- layer.weight_scale = torch.nn.Parameter(max_w_scale,
35
- requires_grad=False)
36
- if self.input_dynamic:
37
- layer.input_scale = None
37
+ # If per tensor, when we have a fused module (e.g. QKV) with per
38
+ # tensor scales (thus N scales being passed to the kernel),
39
+ # requantize so we can always run per tensor
40
+ if self.strategy == QuantizationStrategy.TENSOR:
41
+ max_w_scale, weight = requantize_with_max_scale(
42
+ weight=layer.weight,
43
+ weight_scale=layer.weight_scale,
44
+ logical_widths=layer.logical_widths,
45
+ )
46
+
47
+ layer.weight = Parameter(weight.t(), requires_grad=False)
48
+ layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
49
+
50
+ # If channelwise, scales are already lined up, so just transpose.
51
+ elif self.strategy == QuantizationStrategy.CHANNEL:
52
+ assert self.cutlass_fp8_supported
53
+ weight = layer.weight
54
+ layer.weight = Parameter(weight.t(), requires_grad=False)
55
+
56
+ else:
57
+ raise ValueError(f"Unknown quantization strategy {self.strategy}")
58
+
59
+ # INPUT SCALE
60
+ if self.is_static_input_scheme:
61
+ layer.input_scale = Parameter(layer.input_scale.max(),
62
+ requires_grad=False)
63
else:
39
- layer.input_scale = torch.nn.Parameter(layer.input_scale.max(),
40
- requires_grad=False)
64
+ layer.input_scale = None
65
66
def create_weights(self, layer: torch.nn.Module,
67
output_partition_sizes: List[int],
68
input_size_per_partition: int,
69
params_dtype: torch.dtype, weight_loader: Callable,
70
**kwargs):
47
-
48
- del params_dtype
49
-
71
output_size_per_partition = sum(output_partition_sizes)
72
layer.logical_widths = output_partition_sizes
73
@@ -63,12 +84,17 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
84
})
85
86
# WEIGHT SCALE
66
- weight_scale = create_per_tensor_scale_param(
67
- output_partition_sizes, weight_loader=weight_loader)
87
+ if self.strategy == QuantizationStrategy.CHANNEL:
88
+ weight_scale = create_per_channel_scale_param(
89
+ output_partition_sizes, weight_loader=weight_loader)
90
+ else:
91
+ assert self.strategy == QuantizationStrategy.TENSOR
92
+ weight_scale = create_per_tensor_scale_param(
93
+ output_partition_sizes, weight_loader=weight_loader)
94
layer.register_parameter("weight_scale", weight_scale)
95
96
# INPUT SCALE
71
- if not self.input_dynamic:
97
+ if self.is_static_input_scheme:
98
input_scale = create_per_tensor_scale_param(
99
output_partition_sizes, weight_loader=weight_loader)
100
layer.register_parameter("input_scale", input_scale)
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+10
@@ -9,6 +9,7 @@ from torch.nn import Module
9
class CompressionFormat(Enum):
10
dense = "dense"
11
sparse_bitmask = "sparse-bitmask"
12
+ naive_quantized = "naive-quantized"
13
float_quantized = "float-quantized"
14
int_quantized = "int-quantized"
15
pack_quantized = "pack-quantized"
@@ -76,6 +77,15 @@ class QuantizationArgs(BaseModel):
77
)
78
79
80
+def is_activation_quantization_format(format: str) -> bool:
81
+ _ACTIVATION_QUANTIZATION_FORMATS = [
82
+ CompressionFormat.naive_quantized.value,
83
+ CompressionFormat.int_quantized.value,
84
+ CompressionFormat.float_quantized.value
85
+ ]
86
+ return format in _ACTIVATION_QUANTIZATION_FORMATS
87
+
88
+
89
def find_first_name_or_class_match(
90
name: str,
91
module: Module,