A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[ Misc ] `fp8-marlin` channelwise via `compressed-tensors` (#6524)
Co-authored-by: mgoin <michael@neuralmagic.com>
Robert Shaw committed
Jul 25, 2024 at 09:46 UTC
889da130e747b1382268ed428352f2e73e51a30b
13 files changed
+219
-49
.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-FP8W8.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Qwen2-1.5B-Instruct-FP8W8 -b auto -l 1000 -f 5 -t 1
2
+model_name: "nm-testing/Qwen2-1.5B-Instruct-FP8W8"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.578
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.585
10
+limit: 1000
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+1
@@ -5,3 +5,4 @@ Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
5
Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
6
Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml
7
Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml
8
+Qwen2-1.5B-Instruct-FP8W8.yaml
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+51
-10
@@ -10,7 +10,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10
W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
11
CompressedTensorsScheme, CompressedTensorsUnquantized,
12
CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8Fp8,
13
- CompressedTensorsW8A8Int8, CompressedTensorsWNA16)
13
+ CompressedTensorsW8A8Int8, CompressedTensorsW8A16Fp8,
14
+ CompressedTensorsWNA16)
15
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
16
CompressionFormat, QuantizationArgs, QuantizationStrategy,
17
QuantizationType, find_matched_target, is_activation_quantization_format,
@@ -100,14 +101,18 @@ class CompressedTensorsConfig(QuantizationConfig):
101
def get_config_filenames(cls) -> List[str]:
102
return []
103
103
- def _check_scheme_supported(self, min_capability: int):
104
+ def _check_scheme_supported(self,
105
+ min_capability: int,
106
+ error: bool = True) -> bool:
107
capability = current_platform.get_device_capability()
108
capability = capability[0] * 10 + capability[1]
106
- if capability < min_capability:
109
+ supported = capability >= min_capability
110
+ if error and not supported:
111
raise RuntimeError(
112
"Quantization scheme is not supported for ",
113
f"the current GPU. Min capability: {min_capability}. ",
114
f"Current capability: {capability}.")
115
+ return supported
116
117
def _is_static_tensor_w8a8(self, weight_quant: BaseModel,
118
input_quant: BaseModel) -> bool:
@@ -170,6 +175,29 @@ class CompressedTensorsConfig(QuantizationConfig):
175
# All conditions satisfied.
176
return True
177
178
+ def _is_fp8_w8a16(self, weight_quant: BaseModel,
179
+ input_quant: BaseModel) -> bool:
180
+ # Confirm weights quantized.
181
+ if weight_quant is None:
182
+ return False
183
+
184
+ # Confirm we have floating points.
185
+ if weight_quant.type != QuantizationType.FLOAT:
186
+ return False
187
+
188
+ # Confirm weight scheme is supported.
189
+ is_symmetric_weight = weight_quant.symmetric
190
+ is_static_weight = not weight_quant.dynamic
191
+ is_per_tensor_or_channel_weight = (weight_quant.strategy in [
192
+ QuantizationStrategy.TENSOR, QuantizationStrategy.CHANNEL
193
+ ])
194
+ if not (is_symmetric_weight and is_static_weight
195
+ and is_per_tensor_or_channel_weight):
196
+ return False
197
+
198
+ # All conditions satisfied.
199
+ return True
200
+
201
def _is_wNa16_group_channel(self, weight_quant: BaseModel,
202
input_quant: BaseModel) -> bool:
203
input_quant_none = input_quant is None
@@ -204,9 +232,23 @@ class CompressedTensorsConfig(QuantizationConfig):
232
# Detect If Activation Quantization.
233
if is_activation_quantization_format(self.quant_format):
234
if self._is_fp8_w8a8(weight_quant, input_quant):
207
- return CompressedTensorsW8A8Fp8(
235
+ is_fp8_w8a8_supported = self._check_scheme_supported(
236
+ CompressedTensorsW8A8Fp8.get_min_capability(), error=False)
237
+ if is_fp8_w8a8_supported:
238
+ return CompressedTensorsW8A8Fp8(
239
+ strategy=weight_quant.strategy,
240
+ is_static_input_scheme=(not input_quant.dynamic))
241
+ else:
242
+ return CompressedTensorsW8A16Fp8(
243
+ strategy=weight_quant.strategy,
244
+ is_static_input_scheme=(input_quant
245
+ and not input_quant.dynamic))
246
+
247
+ if self._is_fp8_w8a16(weight_quant, input_quant):
248
+ return CompressedTensorsW8A16Fp8(
249
strategy=weight_quant.strategy,
209
- is_static_input_scheme=(not input_quant.dynamic))
250
+ is_static_input_scheme=(input_quant
251
+ and not input_quant.dynamic))
252
253
if self._is_static_tensor_w8a8(weight_quant, input_quant):
254
return CompressedTensorsW8A8Int8(
@@ -257,11 +299,10 @@ class CompressedTensorsConfig(QuantizationConfig):
299
targets=self.target_scheme_map.keys())
300
301
# Find the quant_scheme
260
- scheme = self.target_scheme_map[matched_target]
261
-
262
- return self._get_scheme_from_parts(
263
- weight_quant=scheme["weights"],
264
- input_quant=scheme["input_activations"])
302
+ scheme_dict = self.target_scheme_map[matched_target]
303
+ scheme = self._get_scheme_from_parts(
304
+ weight_quant=scheme_dict["weights"],
305
+ input_quant=scheme_dict["input_activations"])
306
307
# Raise error if device does not support the scheme
308
# (e.g. fp8 needs ada lovelace)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
+2
@@ -4,6 +4,7 @@ from .compressed_tensors_w4a16_24 import (W4A16SPARSE24_SUPPORTED_BITS,
4
CompressedTensorsW4A16Sparse24)
5
from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
6
from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8
7
+from .compressed_tensors_w8a16_fp8 import CompressedTensorsW8A16Fp8
8
from .compressed_tensors_wNa16 import (WNA16_SUPPORTED_BITS,
9
CompressedTensorsWNA16)
10
@@ -11,6 +12,7 @@ __all__ = [
12
"CompressedTensorsScheme",
13
"CompressedTensorsUnquantized",
14
"CompressedTensorsWNA16",
15
+ "CompressedTensorsW8A16Fp8",
16
"CompressedTensorsW4A16Sparse24",
17
"CompressedTensorsW8A8Int8",
18
"CompressedTensorsW8A8Fp8",
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py
+2
-1
@@ -12,8 +12,9 @@ class CompressedTensorsScheme(ABC):
12
of different quantization schemes supported by CompressedTensors.
13
"""
14
15
+ @classmethod
16
@abstractmethod
16
- def get_min_capability(self) -> int:
17
+ def get_min_capability(cls) -> int:
18
"""
19
Get minimum device capability.
20
"""
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_unquantized.py
+2
-1
@@ -18,7 +18,8 @@ class CompressedTensorsUnquantized(CompressedTensorsScheme):
18
in a linear transformation.
19
"""
20
21
- def get_min_capability(self) -> int:
21
+ @classmethod
22
+ def get_min_capability(cls) -> int:
23
# volta and up
24
return 70
25
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+2
-1
@@ -29,7 +29,8 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
29
raise ValueError(
30
"group_size must be given when using strategy group")
31
32
- def get_min_capability(self) -> int:
32
+ @classmethod
33
+ def get_min_capability(cls) -> int:
34
# ampere + up
35
return 80
36
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py
new
+105
@@ -0,0 +1,105 @@
1
+from typing import Callable, List, Optional
2
+
3
+import torch
4
+
5
+from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
6
+ CompressedTensorsScheme)
7
+from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
8
+ QuantizationStrategy)
9
+from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
10
+ apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
11
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
12
+ convert_to_channelwise, create_per_channel_scale_param,
13
+ create_per_tensor_scale_param)
14
+from vllm.model_executor.utils import set_weight_attrs
15
+
16
+__all__ = ["CompressedTensorsW8A16Fp8"]
17
+
18
+SUPPORTED_STRATEGIES = [
19
+ QuantizationStrategy.CHANNEL, QuantizationStrategy.TENSOR
20
+]
21
+
22
+
23
+class CompressedTensorsW8A16Fp8(CompressedTensorsScheme):
24
+
25
+ def __init__(self, strategy: str, is_static_input_scheme: bool):
26
+ self.strategy = strategy
27
+ self.is_static_input_scheme = is_static_input_scheme
28
+
29
+ @classmethod
30
+ def get_min_capability(cls) -> int:
31
+ # ampere and up
32
+ return 80
33
+
34
+ # W8A8-Fp8 kernels support only per-tensor and per-channel cases.
35
+ # So if we have a fused module (QKV, MLP) with per tensor scales,
36
+ # we expand each scale to its shard's channels.
37
+ def process_weights_after_loading(self, layer) -> None:
38
+ if self.strategy == QuantizationStrategy.TENSOR:
39
+ ws_channelwise = convert_to_channelwise(layer.weight_scale,
40
+ layer.logical_widths)
41
+ layer.weight_scale = torch.nn.Parameter(ws_channelwise,
42
+ requires_grad=False)
43
+
44
+ # Weights must be transposed for marlin
45
+ layer.weight = torch.nn.Parameter(layer.weight.t(),
46
+ requires_grad=False)
47
+
48
+ prepare_fp8_layer_for_marlin(layer, strategy="channel")
49
+
50
+ def create_weights(self, layer: torch.nn.Module, input_size: int,
51
+ output_partition_sizes: List[int],
52
+ input_size_per_partition: int,
53
+ params_dtype: torch.dtype, weight_loader: Callable,
54
+ **kwargs):
55
+
56
+ output_size_per_partition = sum(output_partition_sizes)
57
+ layer.logical_widths = output_partition_sizes
58
+ layer.input_size_per_partition = input_size_per_partition
59
+ layer.output_size_per_partition = output_size_per_partition
60
+ layer.orig_dtype = params_dtype
61
+
62
+ # WEIGHT
63
+ weight = torch.nn.Parameter(torch.empty(output_size_per_partition,
64
+ input_size_per_partition,
65
+ dtype=torch.float8_e4m3fn),
66
+ requires_grad=False)
67
+ layer.register_parameter("weight", weight)
68
+ set_weight_attrs(weight, {
69
+ "input_dim": 1,
70
+ "output_dim": 0,
71
+ "weight_loader": weight_loader,
72
+ })
73
+
74
+ # WEIGHT SCALE
75
+ layer_kwargs = {"weight_loader": weight_loader}
76
+ if self.strategy == QuantizationStrategy.CHANNEL:
77
+ weight_scale = create_per_channel_scale_param(
78
+ output_partition_sizes, **layer_kwargs)
79
+ elif self.strategy == QuantizationStrategy.TENSOR:
80
+ weight_scale = create_per_tensor_scale_param(
81
+ output_partition_sizes, **layer_kwargs)
82
+ else:
83
+ raise ValueError(
84
+ f"Unsupported weight strategy={self.strategy}, "
85
+ f"supported strategies are {SUPPORTED_STRATEGIES}")
86
+ layer.register_parameter("weight_scale", weight_scale)
87
+
88
+ # INPUT SCALE (to deal with converted checkpoints)
89
+ if self.is_static_input_scheme:
90
+ input_scale = create_per_tensor_scale_param(
91
+ output_partition_sizes, **layer_kwargs)
92
+ layer.register_parameter("input_scale", input_scale)
93
+
94
+ def apply_weights(self,
95
+ layer: torch.nn.Module,
96
+ x: torch.Tensor,
97
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
98
+
99
+ return apply_fp8_marlin_linear(input=x,
100
+ weight=layer.weight,
101
+ weight_scale=layer.weight_scale,
102
+ workspace=layer.workspace,
103
+ size_n=layer.output_size_per_partition,
104
+ size_k=layer.input_size_per_partition,
105
+ bias=bias)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+6
-4
@@ -23,7 +23,8 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
23
self.is_static_input_scheme = is_static_input_scheme
24
self.cutlass_fp8_supported = cutlass_fp8_supported()
25
26
- def get_min_capability(self) -> int:
26
+ @classmethod
27
+ def get_min_capability(cls) -> int:
28
# lovelace and up
29
return 89
30
@@ -77,19 +78,20 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
78
})
79
80
# WEIGHT SCALE
81
+ layer_kwargs = {"weight_loader": weight_loader}
82
if self.strategy == QuantizationStrategy.CHANNEL:
83
weight_scale = create_per_channel_scale_param(
82
- output_partition_sizes, weight_loader=weight_loader)
84
+ output_partition_sizes, **layer_kwargs)
85
else:
86
assert self.strategy == QuantizationStrategy.TENSOR
87
weight_scale = create_per_tensor_scale_param(
86
- output_partition_sizes, weight_loader=weight_loader)
88
+ output_partition_sizes, **layer_kwargs)
89
layer.register_parameter("weight_scale", weight_scale)
90
91
# INPUT SCALE
92
if self.is_static_input_scheme:
93
input_scale = create_per_tensor_scale_param(
92
- output_partition_sizes, weight_loader=weight_loader)
94
+ output_partition_sizes, **layer_kwargs)
95
layer.register_parameter("input_scale", input_scale)
96
97
def apply_weights(self,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+10
-9
@@ -19,7 +19,8 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
19
self.strategy = strategy
20
self.is_static_input_scheme = is_static_input_scheme
21
22
- def get_min_capability(self) -> int:
22
+ @classmethod
23
+ def get_min_capability(cls) -> int:
24
# turing and up
25
return 75
26
@@ -68,19 +69,19 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
69
# WEIGHT SCALE
70
layer_kwargs = {"weight_loader": weight_loader}
71
if self.strategy == QuantizationStrategy.CHANNEL:
71
- scale = create_per_channel_scale_param(output_partition_sizes,
72
- **layer_kwargs)
72
+ weight_scale = create_per_channel_scale_param(
73
+ output_partition_sizes, **layer_kwargs)
74
else:
75
assert self.strategy == QuantizationStrategy.TENSOR
75
- scale = create_per_tensor_scale_param(output_partition_sizes,
76
- **layer_kwargs)
77
- layer.register_parameter("weight_scale", scale)
76
+ weight_scale = create_per_tensor_scale_param(
77
+ output_partition_sizes, **layer_kwargs)
78
+ layer.register_parameter("weight_scale", weight_scale)
79
80
# INPUT SCALE
81
if self.is_static_input_scheme:
81
- scale = create_per_tensor_scale_param(output_partition_sizes,
82
- **layer_kwargs)
83
- layer.register_parameter("input_scale", scale)
82
+ input_scale = create_per_tensor_scale_param(
83
+ output_partition_sizes, **layer_kwargs)
84
+ layer.register_parameter("input_scale", input_scale)
85
86
def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
87
bias: Optional[torch.Tensor]) -> torch.Tensor:
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+2
-1
@@ -42,7 +42,8 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
42
group_size=self.group_size,
43
is_sym=True)
44
45
- def get_min_capability(self) -> int:
45
+ @classmethod
46
+ def get_min_capability(cls) -> int:
47
# ampere and up
48
return 80
49
vllm/model_executor/layers/quantization/fp8.py
+22
-11
@@ -18,8 +18,9 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
18
from vllm.model_executor.layers.quantization.utils.quant_utils import (
19
is_layer_skipped)
20
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
21
- all_close_1d, apply_fp8_linear, create_per_tensor_scale_param,
22
- cutlass_fp8_supported, per_tensor_dequantize, requantize_with_max_scale)
21
+ all_close_1d, apply_fp8_linear, convert_to_channelwise,
22
+ create_per_tensor_scale_param, cutlass_fp8_supported,
23
+ per_tensor_dequantize, requantize_with_max_scale)
24
from vllm.model_executor.utils import set_weight_attrs
25
from vllm.platforms import current_platform
26
from vllm.utils import print_warning_once
@@ -179,19 +180,29 @@ class Fp8LinearMethod(LinearMethodBase):
180
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
181
layer.input_scale = None
182
182
- # If checkpoint is fp8, requantize the separately quantized logical
183
- # weights into a single fp8 weight with a single weight scale.
183
+ # If checkpoint is fp8, handle that there are N scales for N
184
+ # shards in a fused module
185
else:
185
- # Dequant -> Quant with max scale.
186
- max_w_scale, weight = requantize_with_max_scale(
187
- weight=layer.weight,
188
- weight_scale=layer.weight_scale,
189
- logical_widths=layer.logical_widths,
190
- )
186
+ # If using marlin (w8a16), kernel uses channelwise weights,
187
+ # so extend the weight scales to be channelwise.
188
+ if self.use_marlin:
189
+ weight = layer.weight
190
+ weight_scale = convert_to_channelwise(layer.weight_scale,
191
+ layer.logical_widths)
192
+
193
+ # If using w8a8, torch._scaled_mm needs per tensor, so
194
+ # requantize the logical shards as a single weight.
195
+ else:
196
+ # Dequant -> Quant with max scale so we can run per tensor.
197
+ weight_scale, weight = requantize_with_max_scale(
198
+ weight=layer.weight,
199
+ weight_scale=layer.weight_scale,
200
+ logical_widths=layer.logical_widths,
201
+ )
202
203
# Update layer with new values.
204
layer.weight = Parameter(weight.t(), requires_grad=False)
194
- layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
205
+ layer.weight_scale = Parameter(weight_scale, requires_grad=False)
206
if self.quant_config.activation_scheme == "static":
207
layer.input_scale = Parameter(layer.input_scale.max(),
208
requires_grad=False)
vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py
+3
-11
@@ -46,7 +46,8 @@ def apply_fp8_marlin_linear(
46
return output.reshape(out_shape)
47
48
49
-def prepare_fp8_layer_for_marlin(layer: torch.nn.Module) -> None:
49
+def prepare_fp8_layer_for_marlin(layer: torch.nn.Module,
50
+ strategy: str = "tensor") -> None:
51
print_warning_once(
52
"Your GPU does not have native support for FP8 computation but "
53
"FP8 quantization is being used. Weight-only FP8 compression will "
@@ -74,16 +75,7 @@ def prepare_fp8_layer_for_marlin(layer: torch.nn.Module) -> None:
75
layer.weight = torch.nn.Parameter(marlin_qweight, requires_grad=False)
76
77
# WEIGHT SCALES
77
- # Currently Marlin doesn't support per-tensor scales, so we
78
- # expand it to channelwise
79
- is_channelwise = (len(layer.weight_scale.shape) > 0
80
- and layer.weight_scale.shape[0] == part_size_n)
81
- if is_channelwise:
82
- scales = layer.weight_scale
83
- else:
84
- scales = layer.weight_scale.repeat(1, part_size_n)
85
- scales = scales.to(layer.orig_dtype).to(device)
86
-
78
+ scales = layer.weight_scale.to(layer.orig_dtype)
79
# Permute scales
80
marlin_scales = marlin_permute_scales(s=scales,
81
size_k=part_size_k,