@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 18fecc3559
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[ Kernel ] Fp8 Channelwise Weight Support (#6487)

Robert Shaw committed Jul 17, 2024 at 23:18 UTC 18fecc3559cad615fd8565d1a001557ec355f285
4 files changed +76 -35
vllm/config.py
+2 -1
@@ -238,7 +238,8 @@ class ModelConfig:
238 f"{self.quantization} quantization is currently not "
239 f"supported in ROCm.")
240 if (self.quantization
241 - not in ("fp8", "marlin", "gptq_marlin_24", "gptq_marlin")):
241 + not in ("fp8", "marlin", "gptq_marlin_24", "gptq_marlin",
242 + "compressed_tensors")):
243 logger.warning(
244 "%s quantization is not fully "
245 "optimized yet. The speed can be slower than "
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+11 -7
@@ -13,7 +13,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
13 CompressedTensorsWNA16)
14 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15 CompressionFormat, QuantizationArgs, QuantizationStrategy,
16 - QuantizationType, find_first_name_or_class_match)
16 + QuantizationType, find_first_name_or_class_match,
17 + is_activation_quantization_format)
18 from vllm.platforms import current_platform
19
20
@@ -132,10 +133,11 @@ class CompressedTensorsConfig(QuantizationConfig):
133 # Confirm weight scheme is supported.
134 is_symmetric_weight = weight_quant.symmetric
135 is_static_weight = not weight_quant.dynamic
135 - is_per_tensor_weight = (
136 - weight_quant.strategy == QuantizationStrategy.TENSOR)
136 + is_per_tensor_or_channel_weight = (weight_quant.strategy in [
137 + QuantizationStrategy.TENSOR, QuantizationStrategy.CHANNEL
138 + ])
139 if not (is_symmetric_weight and is_static_weight
138 - and is_per_tensor_weight):
140 + and is_per_tensor_or_channel_weight):
141 return False
142
143 # Dynamic quantization is always supported if weights supported.
@@ -167,6 +169,7 @@ class CompressedTensorsConfig(QuantizationConfig):
169 def _get_schema(self, weight_quant: BaseModel,
170 input_quant: BaseModel) -> "CompressedTensorsScheme":
171
172 + # Detect If Mixed Precision
173 if self._is_wNa16_group_channel(weight_quant, input_quant):
174 self._check_gptq_and_marlin_can_run()
175 if (self.quant_format == CompressionFormat.marlin_24.value
@@ -182,11 +185,12 @@ class CompressedTensorsConfig(QuantizationConfig):
185 strategy=weight_quant.strategy,
186 group_size=weight_quant.group_size)
187
185 - if (self.quant_format == CompressionFormat.int_quantized.value or
186 - self.quant_format == CompressionFormat.float_quantized.value):
188 + # Detect If Activation Quantization.
189 + if is_activation_quantization_format(self.quant_format):
190 if self._is_fp8_w8a8(weight_quant, input_quant):
191 return CompressedTensorsW8A8Fp8(
189 - input_dynamic=input_quant.dynamic)
192 + strategy=weight_quant.strategy,
193 + is_static_input_scheme=(not input_quant.dynamic))
194
195 if self._is_static_tensor_w8a8(weight_quant, input_quant):
196 return CompressedTensorsW8A8Int8(
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+53 -27
@@ -1,11 +1,15 @@
1 from typing import Callable, List, Optional
2
3 import torch
4 +from torch.nn import Parameter
5
6 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7 CompressedTensorsScheme)
8 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9 + QuantizationStrategy)
10 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
8 - apply_fp8_linear, create_per_tensor_scale_param, cutlass_fp8_supported,
11 + apply_fp8_linear, create_per_channel_scale_param,
12 + create_per_tensor_scale_param, cutlass_fp8_supported,
13 requantize_with_max_scale)
14 from vllm.model_executor.utils import set_weight_attrs
15
@@ -14,39 +18,56 @@ __all__ = ["CompressedTensorsW8A8Fp8"]
18
19 class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
20
17 - def __init__(self, input_dynamic: bool):
18 - self.input_dynamic = input_dynamic
21 + def __init__(self, strategy: str, is_static_input_scheme: bool):
22 + self.strategy = strategy
23 + self.is_static_input_scheme = is_static_input_scheme
24 self.cutlass_fp8_supported = cutlass_fp8_supported()
25
21 - # W8A8-Fp8 kernels support only per-tensor and per-channel cases.
22 - # So if we have a fused module (QKV, MLP) with per tensor scales (thus N
23 - # scales being passed to the kernel), we requantize with a single scale.
26 + # On Lovelace, fail for now if channelwise.
27 + # TODO: (@tms) fallback
28 + if (not self.cutlass_fp8_supported
29 + and self.strategy == QuantizationStrategy.CHANNEL):
30 + raise ValueError(
31 + "Channelwise fp8 quantization requires vLLM's custom "
32 + "cutlass kernels, which are not supported on your device."
33 + "Consider quantizing with per tensor scales or upgrading "
34 + "to Hopper.")
35 +
36 def process_weights_after_loading(self, layer) -> None:
25 - # Dequant -> Quant with max scale.
26 - max_w_scale, weight = requantize_with_max_scale(
27 - weight=layer.weight,
28 - weight_scale=layer.weight_scale,
29 - logical_widths=layer.logical_widths,
30 - )
31 -
32 - # Update layer with new values.
33 - layer.weight = torch.nn.Parameter(weight.t(), requires_grad=False)
34 - layer.weight_scale = torch.nn.Parameter(max_w_scale,
35 - requires_grad=False)
36 - if self.input_dynamic:
37 - layer.input_scale = None
37 + # If per tensor, when we have a fused module (e.g. QKV) with per
38 + # tensor scales (thus N scales being passed to the kernel),
39 + # requantize so we can always run per tensor
40 + if self.strategy == QuantizationStrategy.TENSOR:
41 + max_w_scale, weight = requantize_with_max_scale(
42 + weight=layer.weight,
43 + weight_scale=layer.weight_scale,
44 + logical_widths=layer.logical_widths,
45 + )
46 +
47 + layer.weight = Parameter(weight.t(), requires_grad=False)
48 + layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
49 +
50 + # If channelwise, scales are already lined up, so just transpose.
51 + elif self.strategy == QuantizationStrategy.CHANNEL:
52 + assert self.cutlass_fp8_supported
53 + weight = layer.weight
54 + layer.weight = Parameter(weight.t(), requires_grad=False)
55 +
56 + else:
57 + raise ValueError(f"Unknown quantization strategy {self.strategy}")
58 +
59 + # INPUT SCALE
60 + if self.is_static_input_scheme:
61 + layer.input_scale = Parameter(layer.input_scale.max(),
62 + requires_grad=False)
63 else:
39 - layer.input_scale = torch.nn.Parameter(layer.input_scale.max(),
40 - requires_grad=False)
64 + layer.input_scale = None
65
66 def create_weights(self, layer: torch.nn.Module,
67 output_partition_sizes: List[int],
68 input_size_per_partition: int,
69 params_dtype: torch.dtype, weight_loader: Callable,
70 **kwargs):
47 -
48 - del params_dtype
49 -
71 output_size_per_partition = sum(output_partition_sizes)
72 layer.logical_widths = output_partition_sizes
73
@@ -63,12 +84,17 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
84 })
85
86 # WEIGHT SCALE
66 - weight_scale = create_per_tensor_scale_param(
67 - output_partition_sizes, weight_loader=weight_loader)
87 + if self.strategy == QuantizationStrategy.CHANNEL:
88 + weight_scale = create_per_channel_scale_param(
89 + output_partition_sizes, weight_loader=weight_loader)
90 + else:
91 + assert self.strategy == QuantizationStrategy.TENSOR
92 + weight_scale = create_per_tensor_scale_param(
93 + output_partition_sizes, weight_loader=weight_loader)
94 layer.register_parameter("weight_scale", weight_scale)
95
96 # INPUT SCALE
71 - if not self.input_dynamic:
97 + if self.is_static_input_scheme:
98 input_scale = create_per_tensor_scale_param(
99 output_partition_sizes, weight_loader=weight_loader)
100 layer.register_parameter("input_scale", input_scale)
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+10
@@ -9,6 +9,7 @@ from torch.nn import Module
9 class CompressionFormat(Enum):
10 dense = "dense"
11 sparse_bitmask = "sparse-bitmask"
12 + naive_quantized = "naive-quantized"
13 float_quantized = "float-quantized"
14 int_quantized = "int-quantized"
15 pack_quantized = "pack-quantized"
@@ -76,6 +77,15 @@ class QuantizationArgs(BaseModel):
77 )
78
79
80 +def is_activation_quantization_format(format: str) -> bool:
81 + _ACTIVATION_QUANTIZATION_FORMATS = [
82 + CompressionFormat.naive_quantized.value,
83 + CompressionFormat.int_quantized.value,
84 + CompressionFormat.float_quantized.value
85 + ]
86 + return format in _ACTIVATION_QUANTIZATION_FORMATS
87 +
88 +
89 def find_first_name_or_class_match(
90 name: str,
91 module: Module,