@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 889da130e7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[ Misc ] `fp8-marlin` channelwise via `compressed-tensors` (#6524)

Co-authored-by: mgoin <michael@neuralmagic.com>

Robert Shaw committed Jul 25, 2024 at 09:46 UTC 889da130e747b1382268ed428352f2e73e51a30b
13 files changed +219 -49
.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-FP8W8.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Qwen2-1.5B-Instruct-FP8W8 -b auto -l 1000 -f 5 -t 1
2 +model_name: "nm-testing/Qwen2-1.5B-Instruct-FP8W8"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.578
8 + - name: "exact_match,flexible-extract"
9 + value: 0.585
10 +limit: 1000
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+1
@@ -5,3 +5,4 @@ Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
5 Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
6 Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml
7 Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml
8 +Qwen2-1.5B-Instruct-FP8W8.yaml
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+51 -10
@@ -10,7 +10,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10 W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
11 CompressedTensorsScheme, CompressedTensorsUnquantized,
12 CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8Fp8,
13 - CompressedTensorsW8A8Int8, CompressedTensorsWNA16)
13 + CompressedTensorsW8A8Int8, CompressedTensorsW8A16Fp8,
14 + CompressedTensorsWNA16)
15 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
16 CompressionFormat, QuantizationArgs, QuantizationStrategy,
17 QuantizationType, find_matched_target, is_activation_quantization_format,
@@ -100,14 +101,18 @@ class CompressedTensorsConfig(QuantizationConfig):
101 def get_config_filenames(cls) -> List[str]:
102 return []
103
103 - def _check_scheme_supported(self, min_capability: int):
104 + def _check_scheme_supported(self,
105 + min_capability: int,
106 + error: bool = True) -> bool:
107 capability = current_platform.get_device_capability()
108 capability = capability[0] * 10 + capability[1]
106 - if capability < min_capability:
109 + supported = capability >= min_capability
110 + if error and not supported:
111 raise RuntimeError(
112 "Quantization scheme is not supported for ",
113 f"the current GPU. Min capability: {min_capability}. ",
114 f"Current capability: {capability}.")
115 + return supported
116
117 def _is_static_tensor_w8a8(self, weight_quant: BaseModel,
118 input_quant: BaseModel) -> bool:
@@ -170,6 +175,29 @@ class CompressedTensorsConfig(QuantizationConfig):
175 # All conditions satisfied.
176 return True
177
178 + def _is_fp8_w8a16(self, weight_quant: BaseModel,
179 + input_quant: BaseModel) -> bool:
180 + # Confirm weights quantized.
181 + if weight_quant is None:
182 + return False
183 +
184 + # Confirm we have floating points.
185 + if weight_quant.type != QuantizationType.FLOAT:
186 + return False
187 +
188 + # Confirm weight scheme is supported.
189 + is_symmetric_weight = weight_quant.symmetric
190 + is_static_weight = not weight_quant.dynamic
191 + is_per_tensor_or_channel_weight = (weight_quant.strategy in [
192 + QuantizationStrategy.TENSOR, QuantizationStrategy.CHANNEL
193 + ])
194 + if not (is_symmetric_weight and is_static_weight
195 + and is_per_tensor_or_channel_weight):
196 + return False
197 +
198 + # All conditions satisfied.
199 + return True
200 +
201 def _is_wNa16_group_channel(self, weight_quant: BaseModel,
202 input_quant: BaseModel) -> bool:
203 input_quant_none = input_quant is None
@@ -204,9 +232,23 @@ class CompressedTensorsConfig(QuantizationConfig):
232 # Detect If Activation Quantization.
233 if is_activation_quantization_format(self.quant_format):
234 if self._is_fp8_w8a8(weight_quant, input_quant):
207 - return CompressedTensorsW8A8Fp8(
235 + is_fp8_w8a8_supported = self._check_scheme_supported(
236 + CompressedTensorsW8A8Fp8.get_min_capability(), error=False)
237 + if is_fp8_w8a8_supported:
238 + return CompressedTensorsW8A8Fp8(
239 + strategy=weight_quant.strategy,
240 + is_static_input_scheme=(not input_quant.dynamic))
241 + else:
242 + return CompressedTensorsW8A16Fp8(
243 + strategy=weight_quant.strategy,
244 + is_static_input_scheme=(input_quant
245 + and not input_quant.dynamic))
246 +
247 + if self._is_fp8_w8a16(weight_quant, input_quant):
248 + return CompressedTensorsW8A16Fp8(
249 strategy=weight_quant.strategy,
209 - is_static_input_scheme=(not input_quant.dynamic))
250 + is_static_input_scheme=(input_quant
251 + and not input_quant.dynamic))
252
253 if self._is_static_tensor_w8a8(weight_quant, input_quant):
254 return CompressedTensorsW8A8Int8(
@@ -257,11 +299,10 @@ class CompressedTensorsConfig(QuantizationConfig):
299 targets=self.target_scheme_map.keys())
300
301 # Find the quant_scheme
260 - scheme = self.target_scheme_map[matched_target]
261 -
262 - return self._get_scheme_from_parts(
263 - weight_quant=scheme["weights"],
264 - input_quant=scheme["input_activations"])
302 + scheme_dict = self.target_scheme_map[matched_target]
303 + scheme = self._get_scheme_from_parts(
304 + weight_quant=scheme_dict["weights"],
305 + input_quant=scheme_dict["input_activations"])
306
307 # Raise error if device does not support the scheme
308 # (e.g. fp8 needs ada lovelace)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
+2
@@ -4,6 +4,7 @@ from .compressed_tensors_w4a16_24 import (W4A16SPARSE24_SUPPORTED_BITS,
4 CompressedTensorsW4A16Sparse24)
5 from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
6 from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8
7 +from .compressed_tensors_w8a16_fp8 import CompressedTensorsW8A16Fp8
8 from .compressed_tensors_wNa16 import (WNA16_SUPPORTED_BITS,
9 CompressedTensorsWNA16)
10
@@ -11,6 +12,7 @@ __all__ = [
12 "CompressedTensorsScheme",
13 "CompressedTensorsUnquantized",
14 "CompressedTensorsWNA16",
15 + "CompressedTensorsW8A16Fp8",
16 "CompressedTensorsW4A16Sparse24",
17 "CompressedTensorsW8A8Int8",
18 "CompressedTensorsW8A8Fp8",
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py
+2 -1
@@ -12,8 +12,9 @@ class CompressedTensorsScheme(ABC):
12 of different quantization schemes supported by CompressedTensors.
13 """
14
15 + @classmethod
16 @abstractmethod
16 - def get_min_capability(self) -> int:
17 + def get_min_capability(cls) -> int:
18 """
19 Get minimum device capability.
20 """
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_unquantized.py
+2 -1
@@ -18,7 +18,8 @@ class CompressedTensorsUnquantized(CompressedTensorsScheme):
18 in a linear transformation.
19 """
20
21 - def get_min_capability(self) -> int:
21 + @classmethod
22 + def get_min_capability(cls) -> int:
23 # volta and up
24 return 70
25
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+2 -1
@@ -29,7 +29,8 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
29 raise ValueError(
30 "group_size must be given when using strategy group")
31
32 - def get_min_capability(self) -> int:
32 + @classmethod
33 + def get_min_capability(cls) -> int:
34 # ampere + up
35 return 80
36
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py new
+105
@@ -0,0 +1,105 @@
1 +from typing import Callable, List, Optional
2 +
3 +import torch
4 +
5 +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
6 + CompressedTensorsScheme)
7 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
8 + QuantizationStrategy)
9 +from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
10 + apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
11 +from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
12 + convert_to_channelwise, create_per_channel_scale_param,
13 + create_per_tensor_scale_param)
14 +from vllm.model_executor.utils import set_weight_attrs
15 +
16 +__all__ = ["CompressedTensorsW8A16Fp8"]
17 +
18 +SUPPORTED_STRATEGIES = [
19 + QuantizationStrategy.CHANNEL, QuantizationStrategy.TENSOR
20 +]
21 +
22 +
23 +class CompressedTensorsW8A16Fp8(CompressedTensorsScheme):
24 +
25 + def __init__(self, strategy: str, is_static_input_scheme: bool):
26 + self.strategy = strategy
27 + self.is_static_input_scheme = is_static_input_scheme
28 +
29 + @classmethod
30 + def get_min_capability(cls) -> int:
31 + # ampere and up
32 + return 80
33 +
34 + # W8A8-Fp8 kernels support only per-tensor and per-channel cases.
35 + # So if we have a fused module (QKV, MLP) with per tensor scales,
36 + # we expand each scale to its shard's channels.
37 + def process_weights_after_loading(self, layer) -> None:
38 + if self.strategy == QuantizationStrategy.TENSOR:
39 + ws_channelwise = convert_to_channelwise(layer.weight_scale,
40 + layer.logical_widths)
41 + layer.weight_scale = torch.nn.Parameter(ws_channelwise,
42 + requires_grad=False)
43 +
44 + # Weights must be transposed for marlin
45 + layer.weight = torch.nn.Parameter(layer.weight.t(),
46 + requires_grad=False)
47 +
48 + prepare_fp8_layer_for_marlin(layer, strategy="channel")
49 +
50 + def create_weights(self, layer: torch.nn.Module, input_size: int,
51 + output_partition_sizes: List[int],
52 + input_size_per_partition: int,
53 + params_dtype: torch.dtype, weight_loader: Callable,
54 + **kwargs):
55 +
56 + output_size_per_partition = sum(output_partition_sizes)
57 + layer.logical_widths = output_partition_sizes
58 + layer.input_size_per_partition = input_size_per_partition
59 + layer.output_size_per_partition = output_size_per_partition
60 + layer.orig_dtype = params_dtype
61 +
62 + # WEIGHT
63 + weight = torch.nn.Parameter(torch.empty(output_size_per_partition,
64 + input_size_per_partition,
65 + dtype=torch.float8_e4m3fn),
66 + requires_grad=False)
67 + layer.register_parameter("weight", weight)
68 + set_weight_attrs(weight, {
69 + "input_dim": 1,
70 + "output_dim": 0,
71 + "weight_loader": weight_loader,
72 + })
73 +
74 + # WEIGHT SCALE
75 + layer_kwargs = {"weight_loader": weight_loader}
76 + if self.strategy == QuantizationStrategy.CHANNEL:
77 + weight_scale = create_per_channel_scale_param(
78 + output_partition_sizes, **layer_kwargs)
79 + elif self.strategy == QuantizationStrategy.TENSOR:
80 + weight_scale = create_per_tensor_scale_param(
81 + output_partition_sizes, **layer_kwargs)
82 + else:
83 + raise ValueError(
84 + f"Unsupported weight strategy={self.strategy}, "
85 + f"supported strategies are {SUPPORTED_STRATEGIES}")
86 + layer.register_parameter("weight_scale", weight_scale)
87 +
88 + # INPUT SCALE (to deal with converted checkpoints)
89 + if self.is_static_input_scheme:
90 + input_scale = create_per_tensor_scale_param(
91 + output_partition_sizes, **layer_kwargs)
92 + layer.register_parameter("input_scale", input_scale)
93 +
94 + def apply_weights(self,
95 + layer: torch.nn.Module,
96 + x: torch.Tensor,
97 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
98 +
99 + return apply_fp8_marlin_linear(input=x,
100 + weight=layer.weight,
101 + weight_scale=layer.weight_scale,
102 + workspace=layer.workspace,
103 + size_n=layer.output_size_per_partition,
104 + size_k=layer.input_size_per_partition,
105 + bias=bias)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+6 -4
@@ -23,7 +23,8 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
23 self.is_static_input_scheme = is_static_input_scheme
24 self.cutlass_fp8_supported = cutlass_fp8_supported()
25
26 - def get_min_capability(self) -> int:
26 + @classmethod
27 + def get_min_capability(cls) -> int:
28 # lovelace and up
29 return 89
30
@@ -77,19 +78,20 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
78 })
79
80 # WEIGHT SCALE
81 + layer_kwargs = {"weight_loader": weight_loader}
82 if self.strategy == QuantizationStrategy.CHANNEL:
83 weight_scale = create_per_channel_scale_param(
82 - output_partition_sizes, weight_loader=weight_loader)
84 + output_partition_sizes, **layer_kwargs)
85 else:
86 assert self.strategy == QuantizationStrategy.TENSOR
87 weight_scale = create_per_tensor_scale_param(
86 - output_partition_sizes, weight_loader=weight_loader)
88 + output_partition_sizes, **layer_kwargs)
89 layer.register_parameter("weight_scale", weight_scale)
90
91 # INPUT SCALE
92 if self.is_static_input_scheme:
93 input_scale = create_per_tensor_scale_param(
92 - output_partition_sizes, weight_loader=weight_loader)
94 + output_partition_sizes, **layer_kwargs)
95 layer.register_parameter("input_scale", input_scale)
96
97 def apply_weights(self,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+10 -9
@@ -19,7 +19,8 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
19 self.strategy = strategy
20 self.is_static_input_scheme = is_static_input_scheme
21
22 - def get_min_capability(self) -> int:
22 + @classmethod
23 + def get_min_capability(cls) -> int:
24 # turing and up
25 return 75
26
@@ -68,19 +69,19 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
69 # WEIGHT SCALE
70 layer_kwargs = {"weight_loader": weight_loader}
71 if self.strategy == QuantizationStrategy.CHANNEL:
71 - scale = create_per_channel_scale_param(output_partition_sizes,
72 - **layer_kwargs)
72 + weight_scale = create_per_channel_scale_param(
73 + output_partition_sizes, **layer_kwargs)
74 else:
75 assert self.strategy == QuantizationStrategy.TENSOR
75 - scale = create_per_tensor_scale_param(output_partition_sizes,
76 - **layer_kwargs)
77 - layer.register_parameter("weight_scale", scale)
76 + weight_scale = create_per_tensor_scale_param(
77 + output_partition_sizes, **layer_kwargs)
78 + layer.register_parameter("weight_scale", weight_scale)
79
80 # INPUT SCALE
81 if self.is_static_input_scheme:
81 - scale = create_per_tensor_scale_param(output_partition_sizes,
82 - **layer_kwargs)
83 - layer.register_parameter("input_scale", scale)
82 + input_scale = create_per_tensor_scale_param(
83 + output_partition_sizes, **layer_kwargs)
84 + layer.register_parameter("input_scale", input_scale)
85
86 def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
87 bias: Optional[torch.Tensor]) -> torch.Tensor:
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+2 -1
@@ -42,7 +42,8 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
42 group_size=self.group_size,
43 is_sym=True)
44
45 - def get_min_capability(self) -> int:
45 + @classmethod
46 + def get_min_capability(cls) -> int:
47 # ampere and up
48 return 80
49
vllm/model_executor/layers/quantization/fp8.py
+22 -11
@@ -18,8 +18,9 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
18 from vllm.model_executor.layers.quantization.utils.quant_utils import (
19 is_layer_skipped)
20 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
21 - all_close_1d, apply_fp8_linear, create_per_tensor_scale_param,
22 - cutlass_fp8_supported, per_tensor_dequantize, requantize_with_max_scale)
21 + all_close_1d, apply_fp8_linear, convert_to_channelwise,
22 + create_per_tensor_scale_param, cutlass_fp8_supported,
23 + per_tensor_dequantize, requantize_with_max_scale)
24 from vllm.model_executor.utils import set_weight_attrs
25 from vllm.platforms import current_platform
26 from vllm.utils import print_warning_once
@@ -179,19 +180,29 @@ class Fp8LinearMethod(LinearMethodBase):
180 layer.weight_scale = Parameter(weight_scale, requires_grad=False)
181 layer.input_scale = None
182
182 - # If checkpoint is fp8, requantize the separately quantized logical
183 - # weights into a single fp8 weight with a single weight scale.
183 + # If checkpoint is fp8, handle that there are N scales for N
184 + # shards in a fused module
185 else:
185 - # Dequant -> Quant with max scale.
186 - max_w_scale, weight = requantize_with_max_scale(
187 - weight=layer.weight,
188 - weight_scale=layer.weight_scale,
189 - logical_widths=layer.logical_widths,
190 - )
186 + # If using marlin (w8a16), kernel uses channelwise weights,
187 + # so extend the weight scales to be channelwise.
188 + if self.use_marlin:
189 + weight = layer.weight
190 + weight_scale = convert_to_channelwise(layer.weight_scale,
191 + layer.logical_widths)
192 +
193 + # If using w8a8, torch._scaled_mm needs per tensor, so
194 + # requantize the logical shards as a single weight.
195 + else:
196 + # Dequant -> Quant with max scale so we can run per tensor.
197 + weight_scale, weight = requantize_with_max_scale(
198 + weight=layer.weight,
199 + weight_scale=layer.weight_scale,
200 + logical_widths=layer.logical_widths,
201 + )
202
203 # Update layer with new values.
204 layer.weight = Parameter(weight.t(), requires_grad=False)
194 - layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
205 + layer.weight_scale = Parameter(weight_scale, requires_grad=False)
206 if self.quant_config.activation_scheme == "static":
207 layer.input_scale = Parameter(layer.input_scale.max(),
208 requires_grad=False)
vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py
+3 -11
@@ -46,7 +46,8 @@ def apply_fp8_marlin_linear(
46 return output.reshape(out_shape)
47
48
49 -def prepare_fp8_layer_for_marlin(layer: torch.nn.Module) -> None:
49 +def prepare_fp8_layer_for_marlin(layer: torch.nn.Module,
50 + strategy: str = "tensor") -> None:
51 print_warning_once(
52 "Your GPU does not have native support for FP8 computation but "
53 "FP8 quantization is being used. Weight-only FP8 compression will "
@@ -74,16 +75,7 @@ def prepare_fp8_layer_for_marlin(layer: torch.nn.Module) -> None:
75 layer.weight = torch.nn.Parameter(marlin_qweight, requires_grad=False)
76
77 # WEIGHT SCALES
77 - # Currently Marlin doesn't support per-tensor scales, so we
78 - # expand it to channelwise
79 - is_channelwise = (len(layer.weight_scale.shape) > 0
80 - and layer.weight_scale.shape[0] == part_size_n)
81 - if is_channelwise:
82 - scales = layer.weight_scale
83 - else:
84 - scales = layer.weight_scale.repeat(1, part_size_n)
85 - scales = scales.to(layer.orig_dtype).to(device)
86 -
78 + scales = layer.weight_scale.to(layer.orig_dtype)
79 # Permute scales
80 marlin_scales = marlin_permute_scales(s=scales,
81 size_k=part_size_k,