A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[ Misc ] Support Fp8 via `llm-compressor` (#6110)
Co-authored-by: Robert Shaw <rshaw@neuralmagic>
Robert Shaw committed
Jul 7, 2024 at 16:42 UTC
abfe705a02160db53f4b0cf90c7b016f04291b9c
17 files changed
+603
-372
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test -b 32 -l 250 -f 5 -t 1
2
+model_name: "nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.752
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.752
10
+limit: 250
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FP8.yaml
+1
-1
@@ -1,4 +1,4 @@
1
-# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m neuralmagic/Meta-Llama-3-8B-Instruct-FP8 -b 32 -l 250 -f 5 -t 1
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Meta-Llama-3-8B-Instruct-FP8 -b 32 -l 250 -f 5 -t 1
2
model_name: "neuralmagic/Meta-Llama-3-8B-Instruct-FP8"
3
tasks:
4
- name: "gsm8k"
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Per-Token-Test -b "auto" -l 250 -f 5 -t 1
2
+model_name: "nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Per-Token-Test"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.728
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.728
10
+limit: 250
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+2
@@ -1,2 +1,4 @@
1
Meta-Llama-3-8B-Instruct.yaml
2
Meta-Llama-3-8B-Instruct-FP8.yaml
3
+Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
4
+Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
+1
-1
@@ -46,6 +46,6 @@ while getopts "m:b:l:f:t:" OPT; do
46
done
47
48
lm_eval --model vllm \
49
- --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE \
49
+ --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true \
50
--tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
51
--batch_size $BATCH_SIZE
.buildkite/lm-eval-harness/test_lm_eval_correctness.py
+2
-1
@@ -24,7 +24,8 @@ TP_SIZE = os.environ.get("LM_EVAL_TP_SIZE", 1)
24
25
def launch_lm_eval(eval_config):
26
model_args = f"pretrained={eval_config['model_name']}," \
27
- f"tensor_parallel_size={TP_SIZE}"
27
+ f"tensor_parallel_size={TP_SIZE}," \
28
+ f"add_bos_token=true"
29
30
results = lm_eval.simple_evaluate(
31
model="vllm",
tests/quantization/test_compressed_tensors.py
+27
-5
@@ -9,7 +9,8 @@ import torch
9
from vllm import SamplingParams
10
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
11
CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
12
- CompressedTensorsW8A8, CompressedTensorsWNA16)
12
+ CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
13
+ CompressedTensorsWNA16)
14
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15
QuantizationType)
16
@@ -37,12 +38,11 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
38
CompressedTensorsLinearMethod)
39
assert isinstance(down_proj.quant_method,
40
CompressedTensorsLinearMethod)
40
- assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8)
41
+ assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Int8)
42
43
assert qkv_proj.scheme.strategy == strategy
44
assert qkv_proj.scheme.is_static_input_scheme
44
- expected_type = (torch.int8 if quant_type == QuantizationType.INT else
45
- torch.float8_e4m3fn)
45
+ expected_type = torch.int8
46
47
assert qkv_proj.weight.dtype is expected_type
48
assert o_proj.weight.dtype is expected_type
@@ -79,7 +79,7 @@ def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner, model_args):
79
qkv_proj = layer.self_attn.qkv_proj
80
81
assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
82
- assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8)
82
+ assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Int8)
83
assert not qkv_proj.scheme.is_static_input_scheme
84
assert qkv_proj.scheme.strategy == strategy
85
assert qkv_proj.weight.dtype is torch.int8
@@ -123,3 +123,25 @@ def test_compressed_tensors_w4a16_marlin24(vllm_runner):
123
sampling_params = SamplingParams()
124
output = llm.generate("Hello world!", sampling_params=sampling_params)
125
assert output
126
+
127
+
128
+def test_compressed_tensors_fp8(vllm_runner):
129
+ model_path = "nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test"
130
+ with vllm_runner(model_path) as llm:
131
+ model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
132
+ layer = model.model.layers[0]
133
+
134
+ qkv_proj = layer.self_attn.qkv_proj
135
+
136
+ assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
137
+ assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Fp8)
138
+ assert qkv_proj.weight.dtype is torch.float8_e4m3fn
139
+ assert qkv_proj.input_scale.dtype is torch.float32
140
+ assert qkv_proj.weight_scale.dtype is torch.float32
141
+ # should be scalars after processing
142
+ assert len(qkv_proj.input_scale.shape) == 0
143
+ assert len(qkv_proj.weight_scale.shape) == 0
144
+
145
+ sampling_params = SamplingParams()
146
+ output = llm.generate("Hello world!", sampling_params=sampling_params)
147
+ assert output
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+50
-8
@@ -9,10 +9,11 @@ from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
9
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10
W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
11
CompressedTensorsScheme, CompressedTensorsW4A16Sparse24,
12
- CompressedTensorsW8A8, CompressedTensorsWNA16)
12
+ CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
13
+ CompressedTensorsWNA16)
14
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15
CompressionFormat, QuantizationArgs, QuantizationStrategy,
15
- find_first_name_or_class_match)
16
+ QuantizationType, find_first_name_or_class_match)
17
from vllm.platforms import current_platform
18
19
@@ -117,6 +118,40 @@ class CompressedTensorsConfig(QuantizationConfig):
118
119
return is_8_bits and is_token and is_symmetric and is_dynamic
120
121
+ def _is_fp8_w8a8(self, weight_quant: BaseModel,
122
+ input_quant: BaseModel) -> bool:
123
+ # Confirm weights and activations quantized.
124
+ if weight_quant is None or input_quant is None:
125
+ return False
126
+
127
+ # Confirm we have floating points.
128
+ if not (weight_quant.type == QuantizationType.FLOAT
129
+ and input_quant.type == QuantizationType.FLOAT):
130
+ return False
131
+
132
+ # Confirm weight scheme is supported.
133
+ is_symmetric_weight = weight_quant.symmetric
134
+ is_static_weight = not weight_quant.dynamic
135
+ is_per_tensor_weight = (
136
+ weight_quant.strategy == QuantizationStrategy.TENSOR)
137
+ if not (is_symmetric_weight and is_static_weight
138
+ and is_per_tensor_weight):
139
+ return False
140
+
141
+ # Dynamic quantization is always supported if weights supported.
142
+ if input_quant.dynamic:
143
+ return True
144
+
145
+ # Confirm activation scheme is supported.
146
+ is_symmetric_activation = input_quant.symmetric
147
+ is_per_tensor_activation = (
148
+ input_quant.strategy == QuantizationStrategy.TENSOR)
149
+ if not (is_symmetric_activation and is_per_tensor_activation):
150
+ return False
151
+
152
+ # All conditions satisfied.
153
+ return True
154
+
155
def _is_wNa16_group_channel(self, weight_quant: BaseModel,
156
input_quant: BaseModel) -> bool:
157
input_quant_none = input_quant is None
@@ -147,14 +182,21 @@ class CompressedTensorsConfig(QuantizationConfig):
182
strategy=weight_quant.strategy,
183
group_size=weight_quant.group_size)
184
150
- if self.quant_format == CompressionFormat.int_quantized.value:
185
+ if (self.quant_format == CompressionFormat.int_quantized.value or
186
+ self.quant_format == CompressionFormat.float_quantized.value):
187
+ if self._is_fp8_w8a8(weight_quant, input_quant):
188
+ return CompressedTensorsW8A8Fp8(
189
+ input_dynamic=input_quant.dynamic)
190
+
191
if self._is_static_tensor_w8a8(weight_quant, input_quant):
152
- return CompressedTensorsW8A8(strategy=weight_quant.strategy,
153
- is_static_input_scheme=True)
192
+ return CompressedTensorsW8A8Int8(
193
+ strategy=weight_quant.strategy,
194
+ is_static_input_scheme=True)
195
196
if self._is_dynamic_token_w8a8(weight_quant, input_quant):
156
- return CompressedTensorsW8A8(strategy=weight_quant.strategy,
157
- is_static_input_scheme=False)
197
+ return CompressedTensorsW8A8Int8(
198
+ strategy=weight_quant.strategy,
199
+ is_static_input_scheme=False)
200
201
raise NotImplementedError(
202
"No compressed-tensors compatible scheme was found.")
@@ -187,7 +229,7 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
229
self.quantization_config = quantization_config
230
231
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
190
- return layer.scheme.process_weights_after_loading(layer)
232
+ layer.scheme.process_weights_after_loading(layer)
233
234
def create_weights(self, layer: torch.nn.Module,
235
input_size_per_partition: int,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
+19
-8
@@ -1,8 +1,19 @@
1
-from .compressed_tensors_scheme import CompressedTensorsScheme # noqa: F401
2
-from .compressed_tensors_unquantized import ( # noqa: F401
3
- CompressedTensorsUnquantized)
4
-from .compressed_tensors_w4a16_24 import ( # noqa: F401
5
- W4A16SPARSE24_SUPPORTED_BITS, CompressedTensorsW4A16Sparse24)
6
-from .compressed_tensors_w8a8 import CompressedTensorsW8A8 # noqa: F401
7
-from .compressed_tensors_wNa16 import WNA16_SUPPORTED_BITS # noqa: F401
8
-from .compressed_tensors_wNa16 import CompressedTensorsWNA16 # noqa: F401
1
+from .compressed_tensors_scheme import CompressedTensorsScheme
2
+from .compressed_tensors_unquantized import CompressedTensorsUnquantized
3
+from .compressed_tensors_w4a16_24 import (W4A16SPARSE24_SUPPORTED_BITS,
4
+ CompressedTensorsW4A16Sparse24)
5
+from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
6
+from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8
7
+from .compressed_tensors_wNa16 import (WNA16_SUPPORTED_BITS,
8
+ CompressedTensorsWNA16)
9
+
10
+__all__ = [
11
+ "CompressedTensorsScheme",
12
+ "CompressedTensorsUnquantized",
13
+ "CompressedTensorsWNA16",
14
+ "CompressedTensorsW4A16Sparse24",
15
+ "CompressedTensorsW8A8Int8",
16
+ "CompressedTensorsW8A8Fp8",
17
+ "WNA16_SUPPORTED_BITS",
18
+ "W4A16SPARSE24_SUPPORTED_BITS",
19
+]
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8.py
deleted
-109
@@ -1,109 +0,0 @@
1
-from typing import Callable, List, Tuple, Union
2
-
3
-import torch
4
-from torch.nn import Parameter
5
-
6
-from vllm import _custom_ops as ops
7
-from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
- CompressedTensorsScheme)
9
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
10
- QuantizationStrategy)
11
-from vllm.model_executor.utils import set_weight_attrs
12
-
13
-
14
-class CompressedTensorsW8A8(CompressedTensorsScheme):
15
-
16
- def __init__(self, strategy: str, is_static_input_scheme: bool):
17
- self.strategy = strategy
18
- self.is_static_input_scheme = is_static_input_scheme
19
-
20
- # Cutlass kernels support only per-tensor and per-channel cases.
21
- # So if we have a fused module (QKV, MLP) with per tensor scales (thus N
22
- # scales being passed to the kernel), we convert to the per-channel case.
23
- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
24
- if (self.strategy == QuantizationStrategy.TENSOR
25
- and len(self.logical_widths) > 1):
26
-
27
- # Load the N per-tensor scales into the channelwise buffer.
28
- weight_scale_channel = torch.empty(
29
- (sum(self.logical_widths), 1),
30
- dtype=torch.float32,
31
- device=layer.weight_scale.device)
32
- start = 0
33
- for idx, logical_width in enumerate(self.logical_widths):
34
- end = start + logical_width
35
- weight_scale_channel[start:end, :] = layer.weight_scale[idx]
36
- start = end
37
-
38
- layer.weight_scale = Parameter(weight_scale_channel,
39
- requires_grad=False)
40
-
41
- # transpose weights for cutlass.
42
- weight = layer.weight
43
- layer.weight = Parameter(weight.t(), requires_grad=False)
44
-
45
- def create_weights(self, layer: torch.nn.Module,
46
- output_partition_sizes: List[int],
47
- input_size_per_partition: int,
48
- params_dtype: torch.dtype, weight_loader: Callable,
49
- **kwargs):
50
- self.logical_widths = output_partition_sizes
51
-
52
- # WEIGHT SCALE
53
- shape: Union[Tuple[int], Tuple[int, int]]
54
- if self.strategy == QuantizationStrategy.CHANNEL:
55
- shape = (sum(self.logical_widths), 1)
56
- else:
57
- shape = (len(self.logical_widths), )
58
-
59
- weight_scale = Parameter(torch.empty(*shape, dtype=torch.float32),
60
- requires_grad=False)
61
- layer.register_parameter("weight_scale", weight_scale)
62
- if self.strategy == QuantizationStrategy.CHANNEL:
63
- set_weight_attrs(weight_scale, {
64
- "weight_loader": weight_loader,
65
- "output_dim": 0,
66
- })
67
- else:
68
- set_weight_attrs(weight_scale, {
69
- "weight_loader": weight_loader,
70
- "needs_scalar_to_array": True,
71
- })
72
-
73
- # WEIGHT
74
- weight = Parameter(torch.empty(sum(output_partition_sizes),
75
- input_size_per_partition,
76
- dtype=torch.int8),
77
- requires_grad=False)
78
- layer.register_parameter("weight", weight)
79
- set_weight_attrs(weight, {
80
- "input_dim": 1,
81
- "output_dim": 0,
82
- "weight_loader": weight_loader,
83
- })
84
-
85
- # INPUT SCALE
86
- # Static quantization: load from disk.
87
- if self.is_static_input_scheme:
88
- input_scale = Parameter(torch.empty(1, dtype=torch.float32),
89
- requires_grad=False)
90
- layer.register_parameter("input_scale", input_scale)
91
- set_weight_attrs(input_scale, {
92
- "weight_loader": weight_loader,
93
- "ignore_warning": True,
94
- })
95
- # Dynamic quantization: set to None.
96
- else:
97
- layer.input_scale = None
98
-
99
- def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
100
- # ops.scaled_int8_quant supports both dynamic and static quant.
101
- # * dynamic, layer.input_scale is None and x_scale computed from x.
102
- # * static, layer.input_scale is scalar and x_scale is input_scale.
103
- x_q, x_scale = ops.scaled_int8_quant(x, layer.input_scale)
104
-
105
- return ops.cutlass_scaled_mm(x_q,
106
- layer.weight,
107
- scale_a=x_scale,
108
- scale_b=layer.weight_scale,
109
- out_dtype=x.dtype)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
new
+87
@@ -0,0 +1,87 @@
1
+from typing import Callable, List, Optional
2
+
3
+import torch
4
+
5
+from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
6
+ CompressedTensorsScheme)
7
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
8
+ apply_fp8_linear, create_per_tensor_scale_param, cutlass_fp8_supported,
9
+ requantize_with_max_scale)
10
+from vllm.model_executor.utils import set_weight_attrs
11
+
12
+__all__ = ["CompressedTensorsW8A8Fp8"]
13
+
14
+
15
+class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
16
+
17
+ def __init__(self, input_dynamic: bool):
18
+ self.input_dynamic = input_dynamic
19
+ self.cutlass_fp8_supported = cutlass_fp8_supported()
20
+
21
+ # W8A8-Fp8 kernels support only per-tensor and per-channel cases.
22
+ # So if we have a fused module (QKV, MLP) with per tensor scales (thus N
23
+ # scales being passed to the kernel), we requantize with a single scale.
24
+ def process_weights_after_loading(self, layer) -> None:
25
+ # Dequant -> Quant with max scale.
26
+ max_w_scale, weight = requantize_with_max_scale(
27
+ weight=layer.weight,
28
+ weight_scale=layer.weight_scale,
29
+ logical_widths=layer.logical_widths,
30
+ )
31
+
32
+ # Update layer with new values.
33
+ layer.weight = torch.nn.Parameter(weight.t(), requires_grad=False)
34
+ layer.weight_scale = torch.nn.Parameter(max_w_scale,
35
+ requires_grad=False)
36
+ if self.input_dynamic:
37
+ layer.input_scale = None
38
+ else:
39
+ layer.input_scale = torch.nn.Parameter(layer.input_scale.max(),
40
+ requires_grad=False)
41
+
42
+ def create_weights(self, layer: torch.nn.Module,
43
+ output_partition_sizes: List[int],
44
+ input_size_per_partition: int,
45
+ params_dtype: torch.dtype, weight_loader: Callable,
46
+ **kwargs):
47
+
48
+ del params_dtype
49
+
50
+ output_size_per_partition = sum(output_partition_sizes)
51
+ layer.logical_widths = output_partition_sizes
52
+
53
+ # WEIGHT
54
+ weight = torch.nn.Parameter(torch.empty(output_size_per_partition,
55
+ input_size_per_partition,
56
+ dtype=torch.float8_e4m3fn),
57
+ requires_grad=False)
58
+ layer.register_parameter("weight", weight)
59
+ set_weight_attrs(weight, {
60
+ "input_dim": 1,
61
+ "output_dim": 0,
62
+ "weight_loader": weight_loader,
63
+ })
64
+
65
+ # WEIGHT SCALE
66
+ weight_scale = create_per_tensor_scale_param(
67
+ output_partition_sizes, weight_loader=weight_loader)
68
+ layer.register_parameter("weight_scale", weight_scale)
69
+
70
+ # INPUT SCALE
71
+ if not self.input_dynamic:
72
+ input_scale = create_per_tensor_scale_param(
73
+ output_partition_sizes, weight_loader=weight_loader)
74
+ layer.register_parameter("input_scale", input_scale)
75
+
76
+ def apply_weights(self,
77
+ layer: torch.nn.Module,
78
+ x: torch.Tensor,
79
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
80
+
81
+ return apply_fp8_linear(
82
+ input=x,
83
+ weight=layer.weight,
84
+ weight_scale=layer.weight_scale,
85
+ input_scale=layer.input_scale,
86
+ bias=bias,
87
+ cutlass_fp8_supported=self.cutlass_fp8_supported)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
new
+85
@@ -0,0 +1,85 @@
1
+from typing import Callable, List
2
+
3
+import torch
4
+from torch.nn import Parameter
5
+
6
+from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7
+ CompressedTensorsScheme)
8
+from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9
+ QuantizationStrategy)
10
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11
+ apply_int8_linear, convert_to_channelwise, create_per_channel_scale_param,
12
+ create_per_tensor_scale_param)
13
+from vllm.model_executor.utils import set_weight_attrs
14
+
15
+
16
+class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
17
+
18
+ def __init__(self, strategy: str, is_static_input_scheme: bool):
19
+ self.strategy = strategy
20
+ self.is_static_input_scheme = is_static_input_scheme
21
+
22
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
23
+ # WEIGHT
24
+ # Cutlass kernels need transposed weight.
25
+ weight = layer.weight
26
+ layer.weight = Parameter(weight.t(), requires_grad=False)
27
+
28
+ # WEIGHT SCALE
29
+ # Cutlass kernels support only per-tensor and per-channel.
30
+ # If we have a fused module (QKV, MLP) with per tensor scales (thus N
31
+ # scales being passed to the kernel), convert to the per-channel case.
32
+ is_fused_module = len(self.logical_widths) > 1
33
+ if is_fused_module and self.strategy == QuantizationStrategy.TENSOR:
34
+ ws_channelwise = convert_to_channelwise(layer.weight_scale,
35
+ self.logical_widths)
36
+ layer.weight_scale = Parameter(ws_channelwise, requires_grad=False)
37
+
38
+ # INPUT SCALE
39
+ if self.is_static_input_scheme:
40
+ layer.input_scale = Parameter(layer.input_scale.max(),
41
+ requires_grad=False)
42
+ else:
43
+ layer.input_scale = None
44
+
45
+ def create_weights(self, layer: torch.nn.Module,
46
+ output_partition_sizes: List[int],
47
+ input_size_per_partition: int,
48
+ params_dtype: torch.dtype, weight_loader: Callable,
49
+ **kwargs):
50
+ self.logical_widths = output_partition_sizes
51
+
52
+ # WEIGHT
53
+ weight = Parameter(torch.empty(sum(output_partition_sizes),
54
+ input_size_per_partition,
55
+ dtype=torch.int8),
56
+ requires_grad=False)
57
+ layer.register_parameter("weight", weight)
58
+ set_weight_attrs(weight, {
59
+ "input_dim": 1,
60
+ "output_dim": 0,
61
+ "weight_loader": weight_loader,
62
+ })
63
+
64
+ # WEIGHT SCALE
65
+ layer_kwargs = {"weight_loader": weight_loader}
66
+ if self.strategy == QuantizationStrategy.CHANNEL:
67
+ scale = create_per_channel_scale_param(output_partition_sizes,
68
+ **layer_kwargs)
69
+ else:
70
+ assert self.strategy == QuantizationStrategy.TENSOR
71
+ scale = create_per_tensor_scale_param(output_partition_sizes,
72
+ **layer_kwargs)
73
+ layer.register_parameter("weight_scale", scale)
74
+
75
+ # INPUT SCALE
76
+ if self.is_static_input_scheme:
77
+ scale = create_per_tensor_scale_param(output_partition_sizes,
78
+ **layer_kwargs)
79
+ layer.register_parameter("input_scale", scale)
80
+
81
+ def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
82
+ return apply_int8_linear(input=x,
83
+ weight=layer.weight,
84
+ weight_scale=layer.weight_scale,
85
+ input_scale=layer.input_scale)
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+1
@@ -9,6 +9,7 @@ from torch.nn import Module
9
class CompressionFormat(Enum):
10
dense = "dense"
11
sparse_bitmask = "sparse-bitmask"
12
+ float_quantized = "float-quantized"
13
int_quantized = "int-quantized"
14
pack_quantized = "pack-quantized"
15
marlin_24 = "marlin-24"
vllm/model_executor/layers/quantization/fp8.py
+41
-228
@@ -1,4 +1,4 @@
1
-from typing import Any, Dict, List, Optional, Union
1
+from typing import Any, Dict, List, Optional
2
3
import torch
4
from torch.nn import Module
@@ -11,11 +11,11 @@ from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
11
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig, QuantizeMethodBase)
14
-from vllm.model_executor.layers.quantization.gptq_marlin import (
15
- GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N, GPTQMarlinState,
16
- marlin_permute_scales)
14
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
18
- pack_fp8_to_int32)
15
+ apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
16
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
17
+ all_close_1d, apply_fp8_linear, create_per_tensor_scale_param,
18
+ cutlass_fp8_supported, per_tensor_dequantize, requantize_with_max_scale)
19
from vllm.model_executor.utils import set_weight_attrs
20
from vllm.platforms import current_platform
21
from vllm.utils import print_warning_once
@@ -25,13 +25,6 @@ ACTIVATION_SCHEMES = ["static", "dynamic"]
25
logger = init_logger(__name__)
26
27
28
-def cutlass_fp8_supported() -> bool:
29
- capability = current_platform.get_device_capability()
30
- capability = capability[0] * 10 + capability[1]
31
-
32
- return ops.cutlass_scaled_mm_supports_fp8(capability)
33
-
34
-
28
class Fp8Config(QuantizationConfig):
29
"""Config class for FP8."""
30
@@ -117,23 +110,6 @@ class Fp8LinearMethod(LinearMethodBase):
110
capability = capability[0] * 10 + capability[1]
111
self.use_marlin = capability < 89
112
120
- def _create_scale_param(
121
- self,
122
- scale_name: str,
123
- layer: torch.nn.Module,
124
- output_partition_sizes: List[int],
125
- **extra_weight_attrs,
126
- ) -> None:
127
- scale = Parameter(torch.empty(len(output_partition_sizes),
128
- dtype=torch.float32),
129
- requires_grad=False)
130
- scale[:] = torch.finfo(torch.float8_e4m3fn).min
131
- layer.register_parameter(scale_name, scale)
132
- set_weight_attrs(scale, {
133
- **extra_weight_attrs,
134
- "needs_scalar_to_array": True,
135
- })
136
-
113
def create_weights(
114
self,
115
layer: torch.nn.Module,
@@ -147,7 +123,6 @@ class Fp8LinearMethod(LinearMethodBase):
123
del input_size, output_size
124
output_size_per_partition = sum(output_partition_sizes)
125
150
- layer.process_after_load = True
126
layer.logical_widths = output_partition_sizes
127
128
layer.input_size_per_partition = input_size_per_partition
@@ -173,144 +148,50 @@ class Fp8LinearMethod(LinearMethodBase):
148
# Otherwise, wait until process_weights_after_loading.
149
if self.quant_config.is_checkpoint_fp8_serialized:
150
# WEIGHT SCALE
176
- self._create_scale_param(
177
- scale_name="weight_scale",
178
- layer=layer,
179
- output_partition_sizes=output_partition_sizes,
180
- **extra_weight_attrs)
151
+ scale = create_per_tensor_scale_param(output_partition_sizes,
152
+ **extra_weight_attrs)
153
+ layer.register_parameter("weight_scale", scale)
154
155
# INPUT ACTIVATION SCALE
156
if self.quant_config.activation_scheme == "static":
184
- self._create_scale_param(
185
- scale_name="input_scale",
186
- layer=layer,
187
- output_partition_sizes=output_partition_sizes,
188
- **extra_weight_attrs)
189
-
190
- # For GPUs without FP8 hardware support, we use Marlin for fast
191
- # fused dequantization
192
- if self.use_marlin:
193
- layer.marlin_state = GPTQMarlinState.REPACK
194
-
195
- def prepare_layer_for_marlin(self, layer: Module) -> None:
196
- print_warning_once(
197
- "Your GPU does not have native support for FP8 computation but "
198
- "FP8 quantization is being used. Weight-only FP8 compression will "
199
- "be used leveraging the Marlin kernel. This may degrade "
200
- "performance for compute-heavy workloads.")
201
-
202
- part_size_n = layer.output_size_per_partition
203
- part_size_k = layer.input_size_per_partition
204
-
205
- assert layer.marlin_state == GPTQMarlinState.REPACK
206
- layer.marlin_state = GPTQMarlinState.READY
207
-
208
- device = layer.weight.device
209
-
210
- # WEIGHTS
211
- # Repack weights to gptq format (packed int32 elements)
212
- packed_gptq_qweight = pack_fp8_to_int32(layer.weight)
213
-
214
- # Repack weights to marlin format
215
- marlin_qweight = ops.gptq_marlin_repack(
216
- b_q_weight=packed_gptq_qweight,
217
- perm=torch.empty(0, dtype=torch.int, device=device),
218
- size_k=part_size_k,
219
- size_n=part_size_n,
220
- num_bits=8,
221
- )
222
- layer.weight = Parameter(marlin_qweight, requires_grad=False)
223
-
224
- # WEIGHT SCALES
225
- # Currently Marlin doesn't support per-tensor scales, so we
226
- # expand it to channelwise
227
- scales = layer.weight_scale.repeat(1, part_size_n).to(
228
- layer.orig_dtype).to(device)
229
- # Permute scales
230
- marlin_scales = marlin_permute_scales(
231
- s=scales,
232
- size_k=part_size_k,
233
- size_n=part_size_n,
234
- group_size=-1,
235
- num_bits=8,
236
- )
237
- layer.weight_scale = Parameter(marlin_scales, requires_grad=False)
238
-
239
- # Allocate marlin workspace
240
- max_workspace_size = (
241
- part_size_n // GPTQ_MARLIN_MIN_THREAD_N) * GPTQ_MARLIN_MAX_PARALLEL
242
- workspace = torch.zeros(max_workspace_size,
243
- dtype=torch.int,
244
- device=device,
245
- requires_grad=False)
246
-
247
- layer.workspace = workspace
157
+ scale = create_per_tensor_scale_param(output_partition_sizes,
158
+ **extra_weight_attrs)
159
+ layer.register_parameter("input_scale", scale)
160
161
def process_weights_after_loading(self, layer: Module) -> None:
250
- if (not hasattr(layer, "process_after_load")
251
- or not layer.process_after_load):
252
- return
253
-
254
- # If checkpoint is fp/bf16 (not serialized fp8), quantize the weights.
162
+ # If checkpoint not serialized fp8, quantize the weights.
163
if not self.quant_config.is_checkpoint_fp8_serialized:
164
qweight, weight_scale = ops.scaled_fp8_quant(layer.weight,
165
scale=None)
166
+
167
+ # Update the layer with the new values.
168
layer.weight = Parameter(qweight.t(), requires_grad=False)
169
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
260
- layer.logical_widths = None
170
layer.input_scale = None
262
- if self.use_marlin:
263
- self.prepare_layer_for_marlin(layer)
264
- return
171
172
# If checkpoint is fp8, requantize the separately quantized logical
173
# weights into a single fp8 weight with a single weight scale.
174
else:
269
- # WEIGHT_SCALE / WEIGHT
270
- # Loop over logical weights, requantizing with single scale.
271
- max_w_scale = layer.weight_scale.max()
272
-
273
- # QKV / MLP is fused in the on disk checkpoint if any of the
274
- # weight scales are still set to the default since we initialize
275
- # N weight scales for N shards but we only load 1 weight scale
276
- # from disk in this case. As a result, we skip dequant -> requant
277
- # since we already have quantized QKV together.
278
- # Sample Model with fused checkpoint:
279
- # * nm-testing/Phi-3-mini-128k-instruct-FP8
280
- unfused_module_in_checkpoint = (
281
- layer.weight_scale[-1] > torch.finfo(torch.float8_e4m3fn).min)
282
-
283
- if unfused_module_in_checkpoint:
284
- start = 0
285
- for idx, logical_width in enumerate(layer.logical_widths):
286
- end = start + logical_width
287
- weight_dq = per_tensor_dequantize(
288
- layer.weight[start:end, :], layer.weight_scale[idx])
289
-
290
- layer.weight[start:end, :] = per_tensor_quantize(
291
- weight_dq, layer.weight_scale.max())
292
- start = end
293
- layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
175
+ # Dequant -> Quant with max scale.
176
+ max_w_scale, weight = requantize_with_max_scale(
177
+ weight=layer.weight,
178
+ weight_scale=layer.weight_scale,
179
+ logical_widths=layer.logical_widths,
180
+ )
181
295
- # WEIGHT
296
- # Transpose weight for passing to torch._scaled_mm
297
- weight = layer.weight
182
+ # Update layer with new values.
183
layer.weight = Parameter(weight.t(), requires_grad=False)
299
-
300
- # INPUT ACTIVATION SCALE
301
- # Dynamic: set to None (required input to ops.scaled_fp8_quant).
302
- # Static: set to max of the input_scales (since they are equal).
303
- if self.quant_config.activation_scheme == "dynamic":
304
- layer.input_scale = None
305
- elif self.quant_config.activation_scheme == "static":
184
+ layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
185
+ if self.quant_config.activation_scheme == "static":
186
layer.input_scale = Parameter(layer.input_scale.max(),
187
requires_grad=False)
188
else:
309
- raise ValueError(
310
- f"Unknown scheme {self.quant_config.activation_scheme}")
189
+ layer.input_scale = None
190
312
- if self.use_marlin:
313
- self.prepare_layer_for_marlin(layer)
191
+ if self.use_marlin:
192
+ prepare_fp8_layer_for_marlin(layer)
193
+ # Activations not quantized for marlin.
194
+ del layer.input_scale
195
196
def apply(self,
197
layer: torch.nn.Module,
@@ -318,65 +199,22 @@ class Fp8LinearMethod(LinearMethodBase):
199
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
200
201
if self.use_marlin:
321
- # For GPUs that lack FP8 hardware support, we can leverage the
322
- # Marlin kernel for fast weight-only FP8 quantization
323
-
324
- reshaped_x = x.reshape(-1, x.shape[-1])
325
- out_shape = x.shape[:-1] + (layer.output_size_per_partition, )
326
-
327
- output = ops.fp8_marlin_gemm(
328
- a=reshaped_x,
329
- b_q_weight=layer.weight,
330
- b_scales=layer.weight_scale,
202
+ return apply_fp8_marlin_linear(
203
+ input=x,
204
+ weight=layer.weight,
205
+ weight_scale=layer.weight_scale,
206
workspace=layer.workspace,
332
- num_bits=8,
333
- size_m=reshaped_x.shape[0],
207
size_n=layer.output_size_per_partition,
208
size_k=layer.input_size_per_partition,
336
- )
337
-
338
- if bias is not None:
339
- output.add_(bias) # In-place add
340
-
341
- return output.reshape(out_shape)
342
-
343
- else:
344
-
345
- # ops.scaled_fp8_quant supports both dynamic and static quant.
346
- # If dynamic, layer.input_scale is None and x_scale computed from x
347
- # If static, layer.input_scale is scalar and x_scale is input_scale
209
+ bias=bias)
210
349
- if bias is None and self.cutlass_fp8_supported:
350
- qinput, x_scale = ops.scaled_fp8_quant(x, layer.input_scale)
351
-
352
- # Fused GEMM_DQ
353
- output = ops.cutlass_scaled_mm(
354
- qinput,
355
- layer.weight,
356
- out_dtype=x.dtype,
357
- scale_a=x_scale,
358
- scale_b=layer.weight_scale,
359
- )
360
-
361
- else:
362
- qinput, x_scale = ops.scaled_fp8_quant(x,
363
- layer.input_scale,
364
- batch_dim_padding=17)
365
-
366
- # Fused GEMM_DQ -- note we padded the input above because
367
- # torch._scaled_mm is more performant for matrices with
368
- # batch dimension > 16. Note that this could change
369
- # in the future.
370
- output, _ = torch._scaled_mm(
371
- qinput,
372
- layer.weight,
373
- out_dtype=x.dtype,
374
- scale_a=x_scale,
375
- scale_b=layer.weight_scale,
376
- bias=bias,
377
- )
378
-
379
- return torch.narrow(output, 0, 0, x.shape[0])
211
+ return apply_fp8_linear(
212
+ input=x,
213
+ weight=layer.weight,
214
+ weight_scale=layer.weight_scale,
215
+ input_scale=layer.input_scale,
216
+ bias=bias,
217
+ cutlass_fp8_supported=self.cutlass_fp8_supported)
218
219
220
class Fp8MoEMethod(FusedMoEMethodBase):
@@ -399,8 +237,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
237
intermediate_size: int, params_dtype: torch.dtype,
238
**extra_weight_attrs):
239
402
- layer.process_after_load = True
403
-
240
if self.quant_config.is_checkpoint_fp8_serialized:
241
params_dtype = torch.float8_e4m3fn
242
@@ -465,9 +301,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
301
layer.a2_scale = None
302
303
def process_weights_after_loading(self, layer: Module) -> None:
468
- if (not hasattr(layer, "process_after_load")
469
- or not layer.process_after_load):
470
- return
304
305
# If checkpoint is fp16, quantize in place.
306
if not self.quant_config.is_checkpoint_fp8_serialized:
@@ -531,7 +364,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
364
shard_size, :],
365
layer.w13_scale[expert_id][shard_id])
366
layer.w13_weight[expert_id][
534
- start:start + shard_size, :] = per_tensor_quantize(
367
+ start:start + shard_size, :], _ = ops.scaled_fp8_quant(
368
dq_weight, max_w13_scales[expert_id])
369
start += shard_size
370
@@ -596,23 +429,3 @@ class Fp8KVCacheMethod(QuantizeMethodBase):
429
"cause accuracy issues. Please make sure kv-cache scaling "
430
"factor is available in the fp8 checkpoint.")
431
del layer.kv_scale
599
-
600
-
601
-def per_tensor_quantize(tensor: torch.Tensor,
602
- inv_scale: Union[float, torch.Tensor]) -> torch.Tensor:
603
- finfo = torch.finfo(torch.float8_e4m3fn)
604
- qweight = (tensor / inv_scale).clamp(min=finfo.min, max=finfo.max)
605
- return qweight.to(torch.float8_e4m3fn)
606
-
607
-
608
-def per_tensor_dequantize(
609
- tensor: torch.Tensor, inv_scale: Union[float,
610
- torch.Tensor]) -> torch.Tensor:
611
- fake_qweight = tensor.to(torch.float16)
612
- dq_weight = fake_qweight * inv_scale
613
- return dq_weight
614
-
615
-
616
-def all_close_1d(x: torch.Tensor) -> bool:
617
- assert len(x.shape) == 1
618
- return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0]))
vllm/model_executor/layers/quantization/gptq_marlin.py
+5
-9
@@ -11,20 +11,16 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11
set_weight_attrs)
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig)
14
+from vllm.model_executor.layers.quantization.utils.marlin_utils import (
15
+ GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_K,
16
+ GPTQ_MARLIN_MIN_THREAD_N, GPTQ_MARLIN_SUPPORTED_GROUP_SIZES,
17
+ GPTQ_MARLIN_SUPPORTED_NUM_BITS, GPTQ_MARLIN_SUPPORTED_SYM,
18
+ GPTQ_MARLIN_TILE)
19
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
20
from vllm.platforms import current_platform
21
22
logger = init_logger(__name__)
23
19
-GPTQ_MARLIN_TILE = 16
20
-GPTQ_MARLIN_MIN_THREAD_N = 64
21
-GPTQ_MARLIN_MIN_THREAD_K = 128
22
-GPTQ_MARLIN_MAX_PARALLEL = 16
23
-
24
-GPTQ_MARLIN_SUPPORTED_NUM_BITS = [4, 8]
25
-GPTQ_MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
26
-GPTQ_MARLIN_SUPPORTED_SYM = [True]
27
-
24
25
# Permutations for Marlin scale shuffling
26
def get_scale_perms(num_bits: int):
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+97
-2
@@ -1,9 +1,11 @@
1
"""This file is used for /tests and /benchmarks"""
2
import random
3
+from typing import Optional
4
5
import numpy
6
import torch
7
8
+from vllm import _custom_ops as ops
9
from vllm.model_executor.layers.quantization.utils.format_24 import (
10
mask_creator, sparse_semi_structured_from_dense_cutlass)
11
from vllm.model_executor.layers.quantization.utils.marlin_24_perms import (
@@ -13,8 +15,16 @@ from vllm.model_executor.layers.quantization.utils.marlin_perms import (
15
from vllm.model_executor.layers.quantization.utils.quant_utils import (
16
get_pack_factor, quantize_weights, sort_weights)
17
from vllm.platforms import current_platform
18
+from vllm.utils import print_warning_once
19
17
-MARLIN_TILE = 16
20
+GPTQ_MARLIN_TILE = 16
21
+GPTQ_MARLIN_MIN_THREAD_N = 64
22
+GPTQ_MARLIN_MIN_THREAD_K = 128
23
+GPTQ_MARLIN_MAX_PARALLEL = 16
24
+
25
+GPTQ_MARLIN_SUPPORTED_NUM_BITS = [4, 8]
26
+GPTQ_MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
27
+GPTQ_MARLIN_SUPPORTED_SYM = [True]
28
29
30
def is_marlin_supported():
@@ -22,7 +32,92 @@ def is_marlin_supported():
32
return capability[0] >= 8
33
34
25
-def marlin_permute_weights(q_w, size_k, size_n, perm, tile=MARLIN_TILE):
35
+def apply_fp8_marlin_linear(
36
+ input: torch.Tensor,
37
+ weight: torch.Tensor,
38
+ weight_scale: torch.Tensor,
39
+ workspace: torch.Tensor,
40
+ size_n: int,
41
+ size_k: int,
42
+ bias: Optional[torch.Tensor],
43
+) -> torch.Tensor:
44
+ # For GPUs that lack FP8 hardware support, we can leverage the
45
+ # Marlin kernel for fast weight-only FP8 quantization
46
+
47
+ reshaped_x = input.reshape(-1, input.shape[-1])
48
+ out_shape = input.shape[:-1] + (size_n, )
49
+
50
+ output = ops.fp8_marlin_gemm(
51
+ a=reshaped_x,
52
+ b_q_weight=weight,
53
+ b_scales=weight_scale,
54
+ workspace=workspace,
55
+ num_bits=8,
56
+ size_m=reshaped_x.shape[0],
57
+ size_n=size_n,
58
+ size_k=size_k,
59
+ )
60
+
61
+ if bias is not None:
62
+ output.add_(bias) # In-place add
63
+
64
+ return output.reshape(out_shape)
65
+
66
+
67
+def prepare_fp8_layer_for_marlin(layer: torch.nn.Module) -> None:
68
+ print_warning_once(
69
+ "Your GPU does not have native support for FP8 computation but "
70
+ "FP8 quantization is being used. Weight-only FP8 compression will "
71
+ "be used leveraging the Marlin kernel. This may degrade "
72
+ "performance for compute-heavy workloads.")
73
+
74
+ part_size_n = layer.output_size_per_partition
75
+ part_size_k = layer.input_size_per_partition
76
+
77
+ device = layer.weight.device
78
+
79
+ # WEIGHTS
80
+ # Repack weights to gptq format (packed int32 elements)
81
+ packed_gptq_qweight = pack_fp8_to_int32(layer.weight)
82
+
83
+ # Repack weights to marlin format
84
+ marlin_qweight = ops.gptq_marlin_repack(
85
+ b_q_weight=packed_gptq_qweight,
86
+ perm=torch.empty(0, dtype=torch.int, device=device),
87
+ size_k=part_size_k,
88
+ size_n=part_size_n,
89
+ num_bits=8,
90
+ )
91
+ layer.weight = torch.nn.Parameter(marlin_qweight, requires_grad=False)
92
+
93
+ # WEIGHT SCALES
94
+ # Currently Marlin doesn't support per-tensor scales, so we
95
+ # expand it to channelwise
96
+ scales = layer.weight_scale.repeat(1, part_size_n).to(
97
+ layer.orig_dtype).to(device)
98
+ # Permute scales
99
+ num_bits = 8
100
+ marlin_scales = marlin_permute_scales(
101
+ s=scales,
102
+ size_k=part_size_k,
103
+ size_n=part_size_n,
104
+ group_size=-1,
105
+ scale_perm=marlin_scale_perm[num_bits],
106
+ scale_perm_single=marlin_scale_perm_single[num_bits])
107
+ layer.weight_scale = torch.nn.Parameter(marlin_scales, requires_grad=False)
108
+
109
+ # Allocate marlin workspace
110
+ max_workspace_size = (part_size_n //
111
+ GPTQ_MARLIN_MIN_THREAD_N) * GPTQ_MARLIN_MAX_PARALLEL
112
+ workspace = torch.zeros(max_workspace_size,
113
+ dtype=torch.int,
114
+ device=device,
115
+ requires_grad=False)
116
+
117
+ layer.workspace = workspace
118
+
119
+
120
+def marlin_permute_weights(q_w, size_k, size_n, perm, tile=GPTQ_MARLIN_TILE):
121
assert q_w.shape == (size_k, size_n)
122
assert size_k % tile == 0, f"size_k = {size_k}, tile = {tile}"
123
assert size_n % tile == 0, f"size_k = {size_n}, tile = {tile}"
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
new
+163
@@ -0,0 +1,163 @@
1
+from typing import List, Optional, Tuple, Union
2
+
3
+import torch
4
+from torch.nn import Parameter
5
+
6
+from vllm import _custom_ops as ops
7
+from vllm.model_executor.utils import set_weight_attrs
8
+from vllm.platforms import current_platform
9
+
10
+
11
+def cutlass_fp8_supported() -> bool:
12
+ capability = current_platform.get_device_capability()
13
+ capability = capability[0] * 10 + capability[1]
14
+
15
+ return ops.cutlass_scaled_mm_supports_fp8(capability)
16
+
17
+
18
+def per_tensor_dequantize(
19
+ tensor: torch.Tensor, inv_scale: Union[float,
20
+ torch.Tensor]) -> torch.Tensor:
21
+ fake_qweight = tensor.to(torch.float16)
22
+ dq_weight = fake_qweight * inv_scale
23
+ return dq_weight
24
+
25
+
26
+def all_close_1d(x: torch.Tensor) -> bool:
27
+ assert len(x.shape) == 1
28
+ return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0]))
29
+
30
+
31
+def create_per_tensor_scale_param(
32
+ output_partition_sizes: List[int],
33
+ **extra_weight_attrs,
34
+) -> Parameter:
35
+ scale = Parameter(torch.empty(len(output_partition_sizes),
36
+ dtype=torch.float32),
37
+ requires_grad=False)
38
+ scale[:] = torch.finfo(torch.float32).min
39
+ set_weight_attrs(scale, {
40
+ "needs_scalar_to_array": True,
41
+ **extra_weight_attrs
42
+ })
43
+ return scale
44
+
45
+
46
+def create_per_channel_scale_param(output_partition_sizes: List[int],
47
+ **extra_weight_attrs) -> Parameter:
48
+ scale = Parameter(torch.empty((sum(output_partition_sizes), 1),
49
+ dtype=torch.float32),
50
+ requires_grad=False)
51
+ scale[:] = torch.finfo(torch.float32).min
52
+ set_weight_attrs(scale, {"output_dim": 0, **extra_weight_attrs})
53
+ return scale
54
+
55
+
56
+def convert_to_channelwise(
57
+ weight_scale: torch.Tensor,
58
+ logical_widths: List[int]) -> Tuple[torch.Tensor, torch.Tensor]:
59
+ # Create channelwise buffer
60
+ weight_scale_channel = torch.empty((sum(logical_widths), 1),
61
+ dtype=torch.float32,
62
+ device=weight_scale.device)
63
+
64
+ # Expand each scale to match the size of each logical matrix.
65
+ start = 0
66
+ for idx, logical_width in enumerate(logical_widths):
67
+ end = start + logical_width
68
+ weight_scale_channel[start:end, :] = weight_scale[idx]
69
+ start = end
70
+
71
+ return weight_scale_channel
72
+
73
+
74
+def requantize_with_max_scale(
75
+ weight: torch.Tensor, weight_scale: torch.Tensor,
76
+ logical_widths: List[int]) -> Tuple[torch.Tensor, torch.Tensor]:
77
+ # Max scale to be used for requanitzation.
78
+ max_w_scale = weight_scale.max()
79
+
80
+ # QKV / MLP is fused in the on disk checkpoint if any of the
81
+ # weight scales are still set to the default since we initialize
82
+ # N weight scales for N shards but we only load 1 weight scale
83
+ # from disk in this case. Skip requantization in this case (since)
84
+ # we already are quantized with the single scale.
85
+ # * Sample Model: nm-testing/Phi-3-mini-128k-instruct-FP8
86
+ unfused_module_in_checkpoint = (weight_scale[-1] > torch.finfo(
87
+ torch.float8_e4m3fn).min)
88
+
89
+ # If unfused checkpoint, need requanize with the single scale.
90
+ if unfused_module_in_checkpoint:
91
+ start = 0
92
+ for idx, logical_width in enumerate(logical_widths):
93
+ end = start + logical_width
94
+ weight_dq = per_tensor_dequantize(weight[start:end, :],
95
+ weight_scale[idx])
96
+ weight[start:end, :], _ = ops.scaled_fp8_quant(
97
+ weight_dq, max_w_scale)
98
+ start = end
99
+
100
+ return max_w_scale, weight
101
+
102
+
103
+def apply_fp8_linear(
104
+ input: torch.Tensor,
105
+ weight: torch.Tensor,
106
+ weight_scale: torch.Tensor,
107
+ input_scale: torch.Tensor,
108
+ bias: Optional[torch.Tensor] = None,
109
+ cutlass_fp8_supported: bool = True,
110
+) -> torch.Tensor:
111
+ # ops.scaled_fp8_quant supports both dynamic and static quant.
112
+ # If dynamic, layer.input_scale is None and x_scale computed from x.
113
+ # If static, layer.input_scale is scalar and x_scale is input_scale.
114
+
115
+ if bias is None and cutlass_fp8_supported:
116
+ qinput, x_scale = ops.scaled_fp8_quant(input, input_scale)
117
+
118
+ # Fused GEMM_DQ
119
+ output = ops.cutlass_scaled_mm(qinput,
120
+ weight,
121
+ out_dtype=input.dtype,
122
+ scale_a=x_scale,
123
+ scale_b=weight_scale)
124
+
125
+ else:
126
+ qinput, x_scale = ops.scaled_fp8_quant(input,
127
+ input_scale,
128
+ batch_dim_padding=17)
129
+
130
+ # Fused GEMM_DQ -- note we padded the input above because
131
+ # torch._scaled_mm is more performant for matrices with
132
+ # batch dimension > 16. Note that this could change
133
+ # in the future.
134
+ output, _ = torch._scaled_mm(qinput,
135
+ weight,
136
+ out_dtype=input.dtype,
137
+ scale_a=x_scale,
138
+ scale_b=weight_scale,
139
+ bias=bias)
140
+
141
+ return torch.narrow(output, 0, 0, input.shape[0])
142
+
143
+
144
+def apply_int8_linear(
145
+ input: torch.Tensor,
146
+ weight: torch.Tensor,
147
+ weight_scale: torch.Tensor,
148
+ input_scale: torch.Tensor,
149
+ bias: Optional[torch.Tensor] = None,
150
+):
151
+ if bias is not None:
152
+ raise NotImplementedError("W8A8 with int8 does not yet support bias.")
153
+
154
+ # ops.scaled_int8_quant supports both dynamic and static quant.
155
+ # * dynamic, layer.input_scale is None and x_scale computed from x.
156
+ # * static, layer.input_scale is scalar and x_scale is input_scale.
157
+ x_q, x_scale = ops.scaled_int8_quant(input, input_scale)
158
+
159
+ return ops.cutlass_scaled_mm(x_q,
160
+ weight,
161
+ scale_a=x_scale,
162
+ scale_b=weight_scale,
163
+ out_dtype=input.dtype)