@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / abfe705a02
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[ Misc ] Support Fp8 via `llm-compressor` (#6110)

Co-authored-by: Robert Shaw <rshaw@neuralmagic>

Robert Shaw committed Jul 7, 2024 at 16:42 UTC abfe705a02160db53f4b0cf90c7b016f04291b9c
17 files changed +603 -372
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test -b 32 -l 250 -f 5 -t 1
2 +model_name: "nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.752
8 + - name: "exact_match,flexible-extract"
9 + value: 0.752
10 +limit: 250
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FP8.yaml
+1 -1
@@ -1,4 +1,4 @@
1 -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m neuralmagic/Meta-Llama-3-8B-Instruct-FP8 -b 32 -l 250 -f 5 -t 1
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Meta-Llama-3-8B-Instruct-FP8 -b 32 -l 250 -f 5 -t 1
2 model_name: "neuralmagic/Meta-Llama-3-8B-Instruct-FP8"
3 tasks:
4 - name: "gsm8k"
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Per-Token-Test -b "auto" -l 250 -f 5 -t 1
2 +model_name: "nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Per-Token-Test"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.728
8 + - name: "exact_match,flexible-extract"
9 + value: 0.728
10 +limit: 250
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+2
@@ -1,2 +1,4 @@
1 Meta-Llama-3-8B-Instruct.yaml
2 Meta-Llama-3-8B-Instruct-FP8.yaml
3 +Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
4 +Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
+1 -1
@@ -46,6 +46,6 @@ while getopts "m:b:l:f:t:" OPT; do
46 done
47
48 lm_eval --model vllm \
49 - --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE \
49 + --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true \
50 --tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
51 --batch_size $BATCH_SIZE
.buildkite/lm-eval-harness/test_lm_eval_correctness.py
+2 -1
@@ -24,7 +24,8 @@ TP_SIZE = os.environ.get("LM_EVAL_TP_SIZE", 1)
24
25 def launch_lm_eval(eval_config):
26 model_args = f"pretrained={eval_config['model_name']}," \
27 - f"tensor_parallel_size={TP_SIZE}"
27 + f"tensor_parallel_size={TP_SIZE}," \
28 + f"add_bos_token=true"
29
30 results = lm_eval.simple_evaluate(
31 model="vllm",
tests/quantization/test_compressed_tensors.py
+27 -5
@@ -9,7 +9,8 @@ import torch
9 from vllm import SamplingParams
10 from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
11 CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
12 - CompressedTensorsW8A8, CompressedTensorsWNA16)
12 + CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
13 + CompressedTensorsWNA16)
14 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15 QuantizationType)
16
@@ -37,12 +38,11 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
38 CompressedTensorsLinearMethod)
39 assert isinstance(down_proj.quant_method,
40 CompressedTensorsLinearMethod)
40 - assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8)
41 + assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Int8)
42
43 assert qkv_proj.scheme.strategy == strategy
44 assert qkv_proj.scheme.is_static_input_scheme
44 - expected_type = (torch.int8 if quant_type == QuantizationType.INT else
45 - torch.float8_e4m3fn)
45 + expected_type = torch.int8
46
47 assert qkv_proj.weight.dtype is expected_type
48 assert o_proj.weight.dtype is expected_type
@@ -79,7 +79,7 @@ def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner, model_args):
79 qkv_proj = layer.self_attn.qkv_proj
80
81 assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
82 - assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8)
82 + assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Int8)
83 assert not qkv_proj.scheme.is_static_input_scheme
84 assert qkv_proj.scheme.strategy == strategy
85 assert qkv_proj.weight.dtype is torch.int8
@@ -123,3 +123,25 @@ def test_compressed_tensors_w4a16_marlin24(vllm_runner):
123 sampling_params = SamplingParams()
124 output = llm.generate("Hello world!", sampling_params=sampling_params)
125 assert output
126 +
127 +
128 +def test_compressed_tensors_fp8(vllm_runner):
129 + model_path = "nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test"
130 + with vllm_runner(model_path) as llm:
131 + model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
132 + layer = model.model.layers[0]
133 +
134 + qkv_proj = layer.self_attn.qkv_proj
135 +
136 + assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
137 + assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Fp8)
138 + assert qkv_proj.weight.dtype is torch.float8_e4m3fn
139 + assert qkv_proj.input_scale.dtype is torch.float32
140 + assert qkv_proj.weight_scale.dtype is torch.float32
141 + # should be scalars after processing
142 + assert len(qkv_proj.input_scale.shape) == 0
143 + assert len(qkv_proj.weight_scale.shape) == 0
144 +
145 + sampling_params = SamplingParams()
146 + output = llm.generate("Hello world!", sampling_params=sampling_params)
147 + assert output
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+50 -8
@@ -9,10 +9,11 @@ from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
9 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10 W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
11 CompressedTensorsScheme, CompressedTensorsW4A16Sparse24,
12 - CompressedTensorsW8A8, CompressedTensorsWNA16)
12 + CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
13 + CompressedTensorsWNA16)
14 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15 CompressionFormat, QuantizationArgs, QuantizationStrategy,
15 - find_first_name_or_class_match)
16 + QuantizationType, find_first_name_or_class_match)
17 from vllm.platforms import current_platform
18
19
@@ -117,6 +118,40 @@ class CompressedTensorsConfig(QuantizationConfig):
118
119 return is_8_bits and is_token and is_symmetric and is_dynamic
120
121 + def _is_fp8_w8a8(self, weight_quant: BaseModel,
122 + input_quant: BaseModel) -> bool:
123 + # Confirm weights and activations quantized.
124 + if weight_quant is None or input_quant is None:
125 + return False
126 +
127 + # Confirm we have floating points.
128 + if not (weight_quant.type == QuantizationType.FLOAT
129 + and input_quant.type == QuantizationType.FLOAT):
130 + return False
131 +
132 + # Confirm weight scheme is supported.
133 + is_symmetric_weight = weight_quant.symmetric
134 + is_static_weight = not weight_quant.dynamic
135 + is_per_tensor_weight = (
136 + weight_quant.strategy == QuantizationStrategy.TENSOR)
137 + if not (is_symmetric_weight and is_static_weight
138 + and is_per_tensor_weight):
139 + return False
140 +
141 + # Dynamic quantization is always supported if weights supported.
142 + if input_quant.dynamic:
143 + return True
144 +
145 + # Confirm activation scheme is supported.
146 + is_symmetric_activation = input_quant.symmetric
147 + is_per_tensor_activation = (
148 + input_quant.strategy == QuantizationStrategy.TENSOR)
149 + if not (is_symmetric_activation and is_per_tensor_activation):
150 + return False
151 +
152 + # All conditions satisfied.
153 + return True
154 +
155 def _is_wNa16_group_channel(self, weight_quant: BaseModel,
156 input_quant: BaseModel) -> bool:
157 input_quant_none = input_quant is None
@@ -147,14 +182,21 @@ class CompressedTensorsConfig(QuantizationConfig):
182 strategy=weight_quant.strategy,
183 group_size=weight_quant.group_size)
184
150 - if self.quant_format == CompressionFormat.int_quantized.value:
185 + if (self.quant_format == CompressionFormat.int_quantized.value or
186 + self.quant_format == CompressionFormat.float_quantized.value):
187 + if self._is_fp8_w8a8(weight_quant, input_quant):
188 + return CompressedTensorsW8A8Fp8(
189 + input_dynamic=input_quant.dynamic)
190 +
191 if self._is_static_tensor_w8a8(weight_quant, input_quant):
152 - return CompressedTensorsW8A8(strategy=weight_quant.strategy,
153 - is_static_input_scheme=True)
192 + return CompressedTensorsW8A8Int8(
193 + strategy=weight_quant.strategy,
194 + is_static_input_scheme=True)
195
196 if self._is_dynamic_token_w8a8(weight_quant, input_quant):
156 - return CompressedTensorsW8A8(strategy=weight_quant.strategy,
157 - is_static_input_scheme=False)
197 + return CompressedTensorsW8A8Int8(
198 + strategy=weight_quant.strategy,
199 + is_static_input_scheme=False)
200
201 raise NotImplementedError(
202 "No compressed-tensors compatible scheme was found.")
@@ -187,7 +229,7 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
229 self.quantization_config = quantization_config
230
231 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
190 - return layer.scheme.process_weights_after_loading(layer)
232 + layer.scheme.process_weights_after_loading(layer)
233
234 def create_weights(self, layer: torch.nn.Module,
235 input_size_per_partition: int,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
+19 -8
@@ -1,8 +1,19 @@
1 -from .compressed_tensors_scheme import CompressedTensorsScheme # noqa: F401
2 -from .compressed_tensors_unquantized import ( # noqa: F401
3 - CompressedTensorsUnquantized)
4 -from .compressed_tensors_w4a16_24 import ( # noqa: F401
5 - W4A16SPARSE24_SUPPORTED_BITS, CompressedTensorsW4A16Sparse24)
6 -from .compressed_tensors_w8a8 import CompressedTensorsW8A8 # noqa: F401
7 -from .compressed_tensors_wNa16 import WNA16_SUPPORTED_BITS # noqa: F401
8 -from .compressed_tensors_wNa16 import CompressedTensorsWNA16 # noqa: F401
1 +from .compressed_tensors_scheme import CompressedTensorsScheme
2 +from .compressed_tensors_unquantized import CompressedTensorsUnquantized
3 +from .compressed_tensors_w4a16_24 import (W4A16SPARSE24_SUPPORTED_BITS,
4 + CompressedTensorsW4A16Sparse24)
5 +from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
6 +from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8
7 +from .compressed_tensors_wNa16 import (WNA16_SUPPORTED_BITS,
8 + CompressedTensorsWNA16)
9 +
10 +__all__ = [
11 + "CompressedTensorsScheme",
12 + "CompressedTensorsUnquantized",
13 + "CompressedTensorsWNA16",
14 + "CompressedTensorsW4A16Sparse24",
15 + "CompressedTensorsW8A8Int8",
16 + "CompressedTensorsW8A8Fp8",
17 + "WNA16_SUPPORTED_BITS",
18 + "W4A16SPARSE24_SUPPORTED_BITS",
19 +]
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8.py deleted
-109
@@ -1,109 +0,0 @@
1 -from typing import Callable, List, Tuple, Union
2 -
3 -import torch
4 -from torch.nn import Parameter
5 -
6 -from vllm import _custom_ops as ops
7 -from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 - CompressedTensorsScheme)
9 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
10 - QuantizationStrategy)
11 -from vllm.model_executor.utils import set_weight_attrs
12 -
13 -
14 -class CompressedTensorsW8A8(CompressedTensorsScheme):
15 -
16 - def __init__(self, strategy: str, is_static_input_scheme: bool):
17 - self.strategy = strategy
18 - self.is_static_input_scheme = is_static_input_scheme
19 -
20 - # Cutlass kernels support only per-tensor and per-channel cases.
21 - # So if we have a fused module (QKV, MLP) with per tensor scales (thus N
22 - # scales being passed to the kernel), we convert to the per-channel case.
23 - def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
24 - if (self.strategy == QuantizationStrategy.TENSOR
25 - and len(self.logical_widths) > 1):
26 -
27 - # Load the N per-tensor scales into the channelwise buffer.
28 - weight_scale_channel = torch.empty(
29 - (sum(self.logical_widths), 1),
30 - dtype=torch.float32,
31 - device=layer.weight_scale.device)
32 - start = 0
33 - for idx, logical_width in enumerate(self.logical_widths):
34 - end = start + logical_width
35 - weight_scale_channel[start:end, :] = layer.weight_scale[idx]
36 - start = end
37 -
38 - layer.weight_scale = Parameter(weight_scale_channel,
39 - requires_grad=False)
40 -
41 - # transpose weights for cutlass.
42 - weight = layer.weight
43 - layer.weight = Parameter(weight.t(), requires_grad=False)
44 -
45 - def create_weights(self, layer: torch.nn.Module,
46 - output_partition_sizes: List[int],
47 - input_size_per_partition: int,
48 - params_dtype: torch.dtype, weight_loader: Callable,
49 - **kwargs):
50 - self.logical_widths = output_partition_sizes
51 -
52 - # WEIGHT SCALE
53 - shape: Union[Tuple[int], Tuple[int, int]]
54 - if self.strategy == QuantizationStrategy.CHANNEL:
55 - shape = (sum(self.logical_widths), 1)
56 - else:
57 - shape = (len(self.logical_widths), )
58 -
59 - weight_scale = Parameter(torch.empty(*shape, dtype=torch.float32),
60 - requires_grad=False)
61 - layer.register_parameter("weight_scale", weight_scale)
62 - if self.strategy == QuantizationStrategy.CHANNEL:
63 - set_weight_attrs(weight_scale, {
64 - "weight_loader": weight_loader,
65 - "output_dim": 0,
66 - })
67 - else:
68 - set_weight_attrs(weight_scale, {
69 - "weight_loader": weight_loader,
70 - "needs_scalar_to_array": True,
71 - })
72 -
73 - # WEIGHT
74 - weight = Parameter(torch.empty(sum(output_partition_sizes),
75 - input_size_per_partition,
76 - dtype=torch.int8),
77 - requires_grad=False)
78 - layer.register_parameter("weight", weight)
79 - set_weight_attrs(weight, {
80 - "input_dim": 1,
81 - "output_dim": 0,
82 - "weight_loader": weight_loader,
83 - })
84 -
85 - # INPUT SCALE
86 - # Static quantization: load from disk.
87 - if self.is_static_input_scheme:
88 - input_scale = Parameter(torch.empty(1, dtype=torch.float32),
89 - requires_grad=False)
90 - layer.register_parameter("input_scale", input_scale)
91 - set_weight_attrs(input_scale, {
92 - "weight_loader": weight_loader,
93 - "ignore_warning": True,
94 - })
95 - # Dynamic quantization: set to None.
96 - else:
97 - layer.input_scale = None
98 -
99 - def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
100 - # ops.scaled_int8_quant supports both dynamic and static quant.
101 - # * dynamic, layer.input_scale is None and x_scale computed from x.
102 - # * static, layer.input_scale is scalar and x_scale is input_scale.
103 - x_q, x_scale = ops.scaled_int8_quant(x, layer.input_scale)
104 -
105 - return ops.cutlass_scaled_mm(x_q,
106 - layer.weight,
107 - scale_a=x_scale,
108 - scale_b=layer.weight_scale,
109 - out_dtype=x.dtype)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py new
+87
@@ -0,0 +1,87 @@
1 +from typing import Callable, List, Optional
2 +
3 +import torch
4 +
5 +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
6 + CompressedTensorsScheme)
7 +from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
8 + apply_fp8_linear, create_per_tensor_scale_param, cutlass_fp8_supported,
9 + requantize_with_max_scale)
10 +from vllm.model_executor.utils import set_weight_attrs
11 +
12 +__all__ = ["CompressedTensorsW8A8Fp8"]
13 +
14 +
15 +class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
16 +
17 + def __init__(self, input_dynamic: bool):
18 + self.input_dynamic = input_dynamic
19 + self.cutlass_fp8_supported = cutlass_fp8_supported()
20 +
21 + # W8A8-Fp8 kernels support only per-tensor and per-channel cases.
22 + # So if we have a fused module (QKV, MLP) with per tensor scales (thus N
23 + # scales being passed to the kernel), we requantize with a single scale.
24 + def process_weights_after_loading(self, layer) -> None:
25 + # Dequant -> Quant with max scale.
26 + max_w_scale, weight = requantize_with_max_scale(
27 + weight=layer.weight,
28 + weight_scale=layer.weight_scale,
29 + logical_widths=layer.logical_widths,
30 + )
31 +
32 + # Update layer with new values.
33 + layer.weight = torch.nn.Parameter(weight.t(), requires_grad=False)
34 + layer.weight_scale = torch.nn.Parameter(max_w_scale,
35 + requires_grad=False)
36 + if self.input_dynamic:
37 + layer.input_scale = None
38 + else:
39 + layer.input_scale = torch.nn.Parameter(layer.input_scale.max(),
40 + requires_grad=False)
41 +
42 + def create_weights(self, layer: torch.nn.Module,
43 + output_partition_sizes: List[int],
44 + input_size_per_partition: int,
45 + params_dtype: torch.dtype, weight_loader: Callable,
46 + **kwargs):
47 +
48 + del params_dtype
49 +
50 + output_size_per_partition = sum(output_partition_sizes)
51 + layer.logical_widths = output_partition_sizes
52 +
53 + # WEIGHT
54 + weight = torch.nn.Parameter(torch.empty(output_size_per_partition,
55 + input_size_per_partition,
56 + dtype=torch.float8_e4m3fn),
57 + requires_grad=False)
58 + layer.register_parameter("weight", weight)
59 + set_weight_attrs(weight, {
60 + "input_dim": 1,
61 + "output_dim": 0,
62 + "weight_loader": weight_loader,
63 + })
64 +
65 + # WEIGHT SCALE
66 + weight_scale = create_per_tensor_scale_param(
67 + output_partition_sizes, weight_loader=weight_loader)
68 + layer.register_parameter("weight_scale", weight_scale)
69 +
70 + # INPUT SCALE
71 + if not self.input_dynamic:
72 + input_scale = create_per_tensor_scale_param(
73 + output_partition_sizes, weight_loader=weight_loader)
74 + layer.register_parameter("input_scale", input_scale)
75 +
76 + def apply_weights(self,
77 + layer: torch.nn.Module,
78 + x: torch.Tensor,
79 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
80 +
81 + return apply_fp8_linear(
82 + input=x,
83 + weight=layer.weight,
84 + weight_scale=layer.weight_scale,
85 + input_scale=layer.input_scale,
86 + bias=bias,
87 + cutlass_fp8_supported=self.cutlass_fp8_supported)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py new
+85
@@ -0,0 +1,85 @@
1 +from typing import Callable, List
2 +
3 +import torch
4 +from torch.nn import Parameter
5 +
6 +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7 + CompressedTensorsScheme)
8 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9 + QuantizationStrategy)
10 +from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11 + apply_int8_linear, convert_to_channelwise, create_per_channel_scale_param,
12 + create_per_tensor_scale_param)
13 +from vllm.model_executor.utils import set_weight_attrs
14 +
15 +
16 +class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
17 +
18 + def __init__(self, strategy: str, is_static_input_scheme: bool):
19 + self.strategy = strategy
20 + self.is_static_input_scheme = is_static_input_scheme
21 +
22 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
23 + # WEIGHT
24 + # Cutlass kernels need transposed weight.
25 + weight = layer.weight
26 + layer.weight = Parameter(weight.t(), requires_grad=False)
27 +
28 + # WEIGHT SCALE
29 + # Cutlass kernels support only per-tensor and per-channel.
30 + # If we have a fused module (QKV, MLP) with per tensor scales (thus N
31 + # scales being passed to the kernel), convert to the per-channel case.
32 + is_fused_module = len(self.logical_widths) > 1
33 + if is_fused_module and self.strategy == QuantizationStrategy.TENSOR:
34 + ws_channelwise = convert_to_channelwise(layer.weight_scale,
35 + self.logical_widths)
36 + layer.weight_scale = Parameter(ws_channelwise, requires_grad=False)
37 +
38 + # INPUT SCALE
39 + if self.is_static_input_scheme:
40 + layer.input_scale = Parameter(layer.input_scale.max(),
41 + requires_grad=False)
42 + else:
43 + layer.input_scale = None
44 +
45 + def create_weights(self, layer: torch.nn.Module,
46 + output_partition_sizes: List[int],
47 + input_size_per_partition: int,
48 + params_dtype: torch.dtype, weight_loader: Callable,
49 + **kwargs):
50 + self.logical_widths = output_partition_sizes
51 +
52 + # WEIGHT
53 + weight = Parameter(torch.empty(sum(output_partition_sizes),
54 + input_size_per_partition,
55 + dtype=torch.int8),
56 + requires_grad=False)
57 + layer.register_parameter("weight", weight)
58 + set_weight_attrs(weight, {
59 + "input_dim": 1,
60 + "output_dim": 0,
61 + "weight_loader": weight_loader,
62 + })
63 +
64 + # WEIGHT SCALE
65 + layer_kwargs = {"weight_loader": weight_loader}
66 + if self.strategy == QuantizationStrategy.CHANNEL:
67 + scale = create_per_channel_scale_param(output_partition_sizes,
68 + **layer_kwargs)
69 + else:
70 + assert self.strategy == QuantizationStrategy.TENSOR
71 + scale = create_per_tensor_scale_param(output_partition_sizes,
72 + **layer_kwargs)
73 + layer.register_parameter("weight_scale", scale)
74 +
75 + # INPUT SCALE
76 + if self.is_static_input_scheme:
77 + scale = create_per_tensor_scale_param(output_partition_sizes,
78 + **layer_kwargs)
79 + layer.register_parameter("input_scale", scale)
80 +
81 + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
82 + return apply_int8_linear(input=x,
83 + weight=layer.weight,
84 + weight_scale=layer.weight_scale,
85 + input_scale=layer.input_scale)
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+1
@@ -9,6 +9,7 @@ from torch.nn import Module
9 class CompressionFormat(Enum):
10 dense = "dense"
11 sparse_bitmask = "sparse-bitmask"
12 + float_quantized = "float-quantized"
13 int_quantized = "int-quantized"
14 pack_quantized = "pack-quantized"
15 marlin_24 = "marlin-24"
vllm/model_executor/layers/quantization/fp8.py
+41 -228
@@ -1,4 +1,4 @@
1 -from typing import Any, Dict, List, Optional, Union
1 +from typing import Any, Dict, List, Optional
2
3 import torch
4 from torch.nn import Module
@@ -11,11 +11,11 @@ from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
11 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig, QuantizeMethodBase)
14 -from vllm.model_executor.layers.quantization.gptq_marlin import (
15 - GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N, GPTQMarlinState,
16 - marlin_permute_scales)
14 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
18 - pack_fp8_to_int32)
15 + apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
16 +from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
17 + all_close_1d, apply_fp8_linear, create_per_tensor_scale_param,
18 + cutlass_fp8_supported, per_tensor_dequantize, requantize_with_max_scale)
19 from vllm.model_executor.utils import set_weight_attrs
20 from vllm.platforms import current_platform
21 from vllm.utils import print_warning_once
@@ -25,13 +25,6 @@ ACTIVATION_SCHEMES = ["static", "dynamic"]
25 logger = init_logger(__name__)
26
27
28 -def cutlass_fp8_supported() -> bool:
29 - capability = current_platform.get_device_capability()
30 - capability = capability[0] * 10 + capability[1]
31 -
32 - return ops.cutlass_scaled_mm_supports_fp8(capability)
33 -
34 -
28 class Fp8Config(QuantizationConfig):
29 """Config class for FP8."""
30
@@ -117,23 +110,6 @@ class Fp8LinearMethod(LinearMethodBase):
110 capability = capability[0] * 10 + capability[1]
111 self.use_marlin = capability < 89
112
120 - def _create_scale_param(
121 - self,
122 - scale_name: str,
123 - layer: torch.nn.Module,
124 - output_partition_sizes: List[int],
125 - **extra_weight_attrs,
126 - ) -> None:
127 - scale = Parameter(torch.empty(len(output_partition_sizes),
128 - dtype=torch.float32),
129 - requires_grad=False)
130 - scale[:] = torch.finfo(torch.float8_e4m3fn).min
131 - layer.register_parameter(scale_name, scale)
132 - set_weight_attrs(scale, {
133 - **extra_weight_attrs,
134 - "needs_scalar_to_array": True,
135 - })
136 -
113 def create_weights(
114 self,
115 layer: torch.nn.Module,
@@ -147,7 +123,6 @@ class Fp8LinearMethod(LinearMethodBase):
123 del input_size, output_size
124 output_size_per_partition = sum(output_partition_sizes)
125
150 - layer.process_after_load = True
126 layer.logical_widths = output_partition_sizes
127
128 layer.input_size_per_partition = input_size_per_partition
@@ -173,144 +148,50 @@ class Fp8LinearMethod(LinearMethodBase):
148 # Otherwise, wait until process_weights_after_loading.
149 if self.quant_config.is_checkpoint_fp8_serialized:
150 # WEIGHT SCALE
176 - self._create_scale_param(
177 - scale_name="weight_scale",
178 - layer=layer,
179 - output_partition_sizes=output_partition_sizes,
180 - **extra_weight_attrs)
151 + scale = create_per_tensor_scale_param(output_partition_sizes,
152 + **extra_weight_attrs)
153 + layer.register_parameter("weight_scale", scale)
154
155 # INPUT ACTIVATION SCALE
156 if self.quant_config.activation_scheme == "static":
184 - self._create_scale_param(
185 - scale_name="input_scale",
186 - layer=layer,
187 - output_partition_sizes=output_partition_sizes,
188 - **extra_weight_attrs)
189 -
190 - # For GPUs without FP8 hardware support, we use Marlin for fast
191 - # fused dequantization
192 - if self.use_marlin:
193 - layer.marlin_state = GPTQMarlinState.REPACK
194 -
195 - def prepare_layer_for_marlin(self, layer: Module) -> None:
196 - print_warning_once(
197 - "Your GPU does not have native support for FP8 computation but "
198 - "FP8 quantization is being used. Weight-only FP8 compression will "
199 - "be used leveraging the Marlin kernel. This may degrade "
200 - "performance for compute-heavy workloads.")
201 -
202 - part_size_n = layer.output_size_per_partition
203 - part_size_k = layer.input_size_per_partition
204 -
205 - assert layer.marlin_state == GPTQMarlinState.REPACK
206 - layer.marlin_state = GPTQMarlinState.READY
207 -
208 - device = layer.weight.device
209 -
210 - # WEIGHTS
211 - # Repack weights to gptq format (packed int32 elements)
212 - packed_gptq_qweight = pack_fp8_to_int32(layer.weight)
213 -
214 - # Repack weights to marlin format
215 - marlin_qweight = ops.gptq_marlin_repack(
216 - b_q_weight=packed_gptq_qweight,
217 - perm=torch.empty(0, dtype=torch.int, device=device),
218 - size_k=part_size_k,
219 - size_n=part_size_n,
220 - num_bits=8,
221 - )
222 - layer.weight = Parameter(marlin_qweight, requires_grad=False)
223 -
224 - # WEIGHT SCALES
225 - # Currently Marlin doesn't support per-tensor scales, so we
226 - # expand it to channelwise
227 - scales = layer.weight_scale.repeat(1, part_size_n).to(
228 - layer.orig_dtype).to(device)
229 - # Permute scales
230 - marlin_scales = marlin_permute_scales(
231 - s=scales,
232 - size_k=part_size_k,
233 - size_n=part_size_n,
234 - group_size=-1,
235 - num_bits=8,
236 - )
237 - layer.weight_scale = Parameter(marlin_scales, requires_grad=False)
238 -
239 - # Allocate marlin workspace
240 - max_workspace_size = (
241 - part_size_n // GPTQ_MARLIN_MIN_THREAD_N) * GPTQ_MARLIN_MAX_PARALLEL
242 - workspace = torch.zeros(max_workspace_size,
243 - dtype=torch.int,
244 - device=device,
245 - requires_grad=False)
246 -
247 - layer.workspace = workspace
157 + scale = create_per_tensor_scale_param(output_partition_sizes,
158 + **extra_weight_attrs)
159 + layer.register_parameter("input_scale", scale)
160
161 def process_weights_after_loading(self, layer: Module) -> None:
250 - if (not hasattr(layer, "process_after_load")
251 - or not layer.process_after_load):
252 - return
253 -
254 - # If checkpoint is fp/bf16 (not serialized fp8), quantize the weights.
162 + # If checkpoint not serialized fp8, quantize the weights.
163 if not self.quant_config.is_checkpoint_fp8_serialized:
164 qweight, weight_scale = ops.scaled_fp8_quant(layer.weight,
165 scale=None)
166 +
167 + # Update the layer with the new values.
168 layer.weight = Parameter(qweight.t(), requires_grad=False)
169 layer.weight_scale = Parameter(weight_scale, requires_grad=False)
260 - layer.logical_widths = None
170 layer.input_scale = None
262 - if self.use_marlin:
263 - self.prepare_layer_for_marlin(layer)
264 - return
171
172 # If checkpoint is fp8, requantize the separately quantized logical
173 # weights into a single fp8 weight with a single weight scale.
174 else:
269 - # WEIGHT_SCALE / WEIGHT
270 - # Loop over logical weights, requantizing with single scale.
271 - max_w_scale = layer.weight_scale.max()
272 -
273 - # QKV / MLP is fused in the on disk checkpoint if any of the
274 - # weight scales are still set to the default since we initialize
275 - # N weight scales for N shards but we only load 1 weight scale
276 - # from disk in this case. As a result, we skip dequant -> requant
277 - # since we already have quantized QKV together.
278 - # Sample Model with fused checkpoint:
279 - # * nm-testing/Phi-3-mini-128k-instruct-FP8
280 - unfused_module_in_checkpoint = (
281 - layer.weight_scale[-1] > torch.finfo(torch.float8_e4m3fn).min)
282 -
283 - if unfused_module_in_checkpoint:
284 - start = 0
285 - for idx, logical_width in enumerate(layer.logical_widths):
286 - end = start + logical_width
287 - weight_dq = per_tensor_dequantize(
288 - layer.weight[start:end, :], layer.weight_scale[idx])
289 -
290 - layer.weight[start:end, :] = per_tensor_quantize(
291 - weight_dq, layer.weight_scale.max())
292 - start = end
293 - layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
175 + # Dequant -> Quant with max scale.
176 + max_w_scale, weight = requantize_with_max_scale(
177 + weight=layer.weight,
178 + weight_scale=layer.weight_scale,
179 + logical_widths=layer.logical_widths,
180 + )
181
295 - # WEIGHT
296 - # Transpose weight for passing to torch._scaled_mm
297 - weight = layer.weight
182 + # Update layer with new values.
183 layer.weight = Parameter(weight.t(), requires_grad=False)
299 -
300 - # INPUT ACTIVATION SCALE
301 - # Dynamic: set to None (required input to ops.scaled_fp8_quant).
302 - # Static: set to max of the input_scales (since they are equal).
303 - if self.quant_config.activation_scheme == "dynamic":
304 - layer.input_scale = None
305 - elif self.quant_config.activation_scheme == "static":
184 + layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
185 + if self.quant_config.activation_scheme == "static":
186 layer.input_scale = Parameter(layer.input_scale.max(),
187 requires_grad=False)
188 else:
309 - raise ValueError(
310 - f"Unknown scheme {self.quant_config.activation_scheme}")
189 + layer.input_scale = None
190
312 - if self.use_marlin:
313 - self.prepare_layer_for_marlin(layer)
191 + if self.use_marlin:
192 + prepare_fp8_layer_for_marlin(layer)
193 + # Activations not quantized for marlin.
194 + del layer.input_scale
195
196 def apply(self,
197 layer: torch.nn.Module,
@@ -318,65 +199,22 @@ class Fp8LinearMethod(LinearMethodBase):
199 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
200
201 if self.use_marlin:
321 - # For GPUs that lack FP8 hardware support, we can leverage the
322 - # Marlin kernel for fast weight-only FP8 quantization
323 -
324 - reshaped_x = x.reshape(-1, x.shape[-1])
325 - out_shape = x.shape[:-1] + (layer.output_size_per_partition, )
326 -
327 - output = ops.fp8_marlin_gemm(
328 - a=reshaped_x,
329 - b_q_weight=layer.weight,
330 - b_scales=layer.weight_scale,
202 + return apply_fp8_marlin_linear(
203 + input=x,
204 + weight=layer.weight,
205 + weight_scale=layer.weight_scale,
206 workspace=layer.workspace,
332 - num_bits=8,
333 - size_m=reshaped_x.shape[0],
207 size_n=layer.output_size_per_partition,
208 size_k=layer.input_size_per_partition,
336 - )
337 -
338 - if bias is not None:
339 - output.add_(bias) # In-place add
340 -
341 - return output.reshape(out_shape)
342 -
343 - else:
344 -
345 - # ops.scaled_fp8_quant supports both dynamic and static quant.
346 - # If dynamic, layer.input_scale is None and x_scale computed from x
347 - # If static, layer.input_scale is scalar and x_scale is input_scale
209 + bias=bias)
210
349 - if bias is None and self.cutlass_fp8_supported:
350 - qinput, x_scale = ops.scaled_fp8_quant(x, layer.input_scale)
351 -
352 - # Fused GEMM_DQ
353 - output = ops.cutlass_scaled_mm(
354 - qinput,
355 - layer.weight,
356 - out_dtype=x.dtype,
357 - scale_a=x_scale,
358 - scale_b=layer.weight_scale,
359 - )
360 -
361 - else:
362 - qinput, x_scale = ops.scaled_fp8_quant(x,
363 - layer.input_scale,
364 - batch_dim_padding=17)
365 -
366 - # Fused GEMM_DQ -- note we padded the input above because
367 - # torch._scaled_mm is more performant for matrices with
368 - # batch dimension > 16. Note that this could change
369 - # in the future.
370 - output, _ = torch._scaled_mm(
371 - qinput,
372 - layer.weight,
373 - out_dtype=x.dtype,
374 - scale_a=x_scale,
375 - scale_b=layer.weight_scale,
376 - bias=bias,
377 - )
378 -
379 - return torch.narrow(output, 0, 0, x.shape[0])
211 + return apply_fp8_linear(
212 + input=x,
213 + weight=layer.weight,
214 + weight_scale=layer.weight_scale,
215 + input_scale=layer.input_scale,
216 + bias=bias,
217 + cutlass_fp8_supported=self.cutlass_fp8_supported)
218
219
220 class Fp8MoEMethod(FusedMoEMethodBase):
@@ -399,8 +237,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
237 intermediate_size: int, params_dtype: torch.dtype,
238 **extra_weight_attrs):
239
402 - layer.process_after_load = True
403 -
240 if self.quant_config.is_checkpoint_fp8_serialized:
241 params_dtype = torch.float8_e4m3fn
242
@@ -465,9 +301,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
301 layer.a2_scale = None
302
303 def process_weights_after_loading(self, layer: Module) -> None:
468 - if (not hasattr(layer, "process_after_load")
469 - or not layer.process_after_load):
470 - return
304
305 # If checkpoint is fp16, quantize in place.
306 if not self.quant_config.is_checkpoint_fp8_serialized:
@@ -531,7 +364,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
364 shard_size, :],
365 layer.w13_scale[expert_id][shard_id])
366 layer.w13_weight[expert_id][
534 - start:start + shard_size, :] = per_tensor_quantize(
367 + start:start + shard_size, :], _ = ops.scaled_fp8_quant(
368 dq_weight, max_w13_scales[expert_id])
369 start += shard_size
370
@@ -596,23 +429,3 @@ class Fp8KVCacheMethod(QuantizeMethodBase):
429 "cause accuracy issues. Please make sure kv-cache scaling "
430 "factor is available in the fp8 checkpoint.")
431 del layer.kv_scale
599 -
600 -
601 -def per_tensor_quantize(tensor: torch.Tensor,
602 - inv_scale: Union[float, torch.Tensor]) -> torch.Tensor:
603 - finfo = torch.finfo(torch.float8_e4m3fn)
604 - qweight = (tensor / inv_scale).clamp(min=finfo.min, max=finfo.max)
605 - return qweight.to(torch.float8_e4m3fn)
606 -
607 -
608 -def per_tensor_dequantize(
609 - tensor: torch.Tensor, inv_scale: Union[float,
610 - torch.Tensor]) -> torch.Tensor:
611 - fake_qweight = tensor.to(torch.float16)
612 - dq_weight = fake_qweight * inv_scale
613 - return dq_weight
614 -
615 -
616 -def all_close_1d(x: torch.Tensor) -> bool:
617 - assert len(x.shape) == 1
618 - return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0]))
vllm/model_executor/layers/quantization/gptq_marlin.py
+5 -9
@@ -11,20 +11,16 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11 set_weight_attrs)
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig)
14 +from vllm.model_executor.layers.quantization.utils.marlin_utils import (
15 + GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_K,
16 + GPTQ_MARLIN_MIN_THREAD_N, GPTQ_MARLIN_SUPPORTED_GROUP_SIZES,
17 + GPTQ_MARLIN_SUPPORTED_NUM_BITS, GPTQ_MARLIN_SUPPORTED_SYM,
18 + GPTQ_MARLIN_TILE)
19 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
20 from vllm.platforms import current_platform
21
22 logger = init_logger(__name__)
23
19 -GPTQ_MARLIN_TILE = 16
20 -GPTQ_MARLIN_MIN_THREAD_N = 64
21 -GPTQ_MARLIN_MIN_THREAD_K = 128
22 -GPTQ_MARLIN_MAX_PARALLEL = 16
23 -
24 -GPTQ_MARLIN_SUPPORTED_NUM_BITS = [4, 8]
25 -GPTQ_MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
26 -GPTQ_MARLIN_SUPPORTED_SYM = [True]
27 -
24
25 # Permutations for Marlin scale shuffling
26 def get_scale_perms(num_bits: int):
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+97 -2
@@ -1,9 +1,11 @@
1 """This file is used for /tests and /benchmarks"""
2 import random
3 +from typing import Optional
4
5 import numpy
6 import torch
7
8 +from vllm import _custom_ops as ops
9 from vllm.model_executor.layers.quantization.utils.format_24 import (
10 mask_creator, sparse_semi_structured_from_dense_cutlass)
11 from vllm.model_executor.layers.quantization.utils.marlin_24_perms import (
@@ -13,8 +15,16 @@ from vllm.model_executor.layers.quantization.utils.marlin_perms import (
15 from vllm.model_executor.layers.quantization.utils.quant_utils import (
16 get_pack_factor, quantize_weights, sort_weights)
17 from vllm.platforms import current_platform
18 +from vllm.utils import print_warning_once
19
17 -MARLIN_TILE = 16
20 +GPTQ_MARLIN_TILE = 16
21 +GPTQ_MARLIN_MIN_THREAD_N = 64
22 +GPTQ_MARLIN_MIN_THREAD_K = 128
23 +GPTQ_MARLIN_MAX_PARALLEL = 16
24 +
25 +GPTQ_MARLIN_SUPPORTED_NUM_BITS = [4, 8]
26 +GPTQ_MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
27 +GPTQ_MARLIN_SUPPORTED_SYM = [True]
28
29
30 def is_marlin_supported():
@@ -22,7 +32,92 @@ def is_marlin_supported():
32 return capability[0] >= 8
33
34
25 -def marlin_permute_weights(q_w, size_k, size_n, perm, tile=MARLIN_TILE):
35 +def apply_fp8_marlin_linear(
36 + input: torch.Tensor,
37 + weight: torch.Tensor,
38 + weight_scale: torch.Tensor,
39 + workspace: torch.Tensor,
40 + size_n: int,
41 + size_k: int,
42 + bias: Optional[torch.Tensor],
43 +) -> torch.Tensor:
44 + # For GPUs that lack FP8 hardware support, we can leverage the
45 + # Marlin kernel for fast weight-only FP8 quantization
46 +
47 + reshaped_x = input.reshape(-1, input.shape[-1])
48 + out_shape = input.shape[:-1] + (size_n, )
49 +
50 + output = ops.fp8_marlin_gemm(
51 + a=reshaped_x,
52 + b_q_weight=weight,
53 + b_scales=weight_scale,
54 + workspace=workspace,
55 + num_bits=8,
56 + size_m=reshaped_x.shape[0],
57 + size_n=size_n,
58 + size_k=size_k,
59 + )
60 +
61 + if bias is not None:
62 + output.add_(bias) # In-place add
63 +
64 + return output.reshape(out_shape)
65 +
66 +
67 +def prepare_fp8_layer_for_marlin(layer: torch.nn.Module) -> None:
68 + print_warning_once(
69 + "Your GPU does not have native support for FP8 computation but "
70 + "FP8 quantization is being used. Weight-only FP8 compression will "
71 + "be used leveraging the Marlin kernel. This may degrade "
72 + "performance for compute-heavy workloads.")
73 +
74 + part_size_n = layer.output_size_per_partition
75 + part_size_k = layer.input_size_per_partition
76 +
77 + device = layer.weight.device
78 +
79 + # WEIGHTS
80 + # Repack weights to gptq format (packed int32 elements)
81 + packed_gptq_qweight = pack_fp8_to_int32(layer.weight)
82 +
83 + # Repack weights to marlin format
84 + marlin_qweight = ops.gptq_marlin_repack(
85 + b_q_weight=packed_gptq_qweight,
86 + perm=torch.empty(0, dtype=torch.int, device=device),
87 + size_k=part_size_k,
88 + size_n=part_size_n,
89 + num_bits=8,
90 + )
91 + layer.weight = torch.nn.Parameter(marlin_qweight, requires_grad=False)
92 +
93 + # WEIGHT SCALES
94 + # Currently Marlin doesn't support per-tensor scales, so we
95 + # expand it to channelwise
96 + scales = layer.weight_scale.repeat(1, part_size_n).to(
97 + layer.orig_dtype).to(device)
98 + # Permute scales
99 + num_bits = 8
100 + marlin_scales = marlin_permute_scales(
101 + s=scales,
102 + size_k=part_size_k,
103 + size_n=part_size_n,
104 + group_size=-1,
105 + scale_perm=marlin_scale_perm[num_bits],
106 + scale_perm_single=marlin_scale_perm_single[num_bits])
107 + layer.weight_scale = torch.nn.Parameter(marlin_scales, requires_grad=False)
108 +
109 + # Allocate marlin workspace
110 + max_workspace_size = (part_size_n //
111 + GPTQ_MARLIN_MIN_THREAD_N) * GPTQ_MARLIN_MAX_PARALLEL
112 + workspace = torch.zeros(max_workspace_size,
113 + dtype=torch.int,
114 + device=device,
115 + requires_grad=False)
116 +
117 + layer.workspace = workspace
118 +
119 +
120 +def marlin_permute_weights(q_w, size_k, size_n, perm, tile=GPTQ_MARLIN_TILE):
121 assert q_w.shape == (size_k, size_n)
122 assert size_k % tile == 0, f"size_k = {size_k}, tile = {tile}"
123 assert size_n % tile == 0, f"size_k = {size_n}, tile = {tile}"
vllm/model_executor/layers/quantization/utils/w8a8_utils.py new
+163
@@ -0,0 +1,163 @@
1 +from typing import List, Optional, Tuple, Union
2 +
3 +import torch
4 +from torch.nn import Parameter
5 +
6 +from vllm import _custom_ops as ops
7 +from vllm.model_executor.utils import set_weight_attrs
8 +from vllm.platforms import current_platform
9 +
10 +
11 +def cutlass_fp8_supported() -> bool:
12 + capability = current_platform.get_device_capability()
13 + capability = capability[0] * 10 + capability[1]
14 +
15 + return ops.cutlass_scaled_mm_supports_fp8(capability)
16 +
17 +
18 +def per_tensor_dequantize(
19 + tensor: torch.Tensor, inv_scale: Union[float,
20 + torch.Tensor]) -> torch.Tensor:
21 + fake_qweight = tensor.to(torch.float16)
22 + dq_weight = fake_qweight * inv_scale
23 + return dq_weight
24 +
25 +
26 +def all_close_1d(x: torch.Tensor) -> bool:
27 + assert len(x.shape) == 1
28 + return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0]))
29 +
30 +
31 +def create_per_tensor_scale_param(
32 + output_partition_sizes: List[int],
33 + **extra_weight_attrs,
34 +) -> Parameter:
35 + scale = Parameter(torch.empty(len(output_partition_sizes),
36 + dtype=torch.float32),
37 + requires_grad=False)
38 + scale[:] = torch.finfo(torch.float32).min
39 + set_weight_attrs(scale, {
40 + "needs_scalar_to_array": True,
41 + **extra_weight_attrs
42 + })
43 + return scale
44 +
45 +
46 +def create_per_channel_scale_param(output_partition_sizes: List[int],
47 + **extra_weight_attrs) -> Parameter:
48 + scale = Parameter(torch.empty((sum(output_partition_sizes), 1),
49 + dtype=torch.float32),
50 + requires_grad=False)
51 + scale[:] = torch.finfo(torch.float32).min
52 + set_weight_attrs(scale, {"output_dim": 0, **extra_weight_attrs})
53 + return scale
54 +
55 +
56 +def convert_to_channelwise(
57 + weight_scale: torch.Tensor,
58 + logical_widths: List[int]) -> Tuple[torch.Tensor, torch.Tensor]:
59 + # Create channelwise buffer
60 + weight_scale_channel = torch.empty((sum(logical_widths), 1),
61 + dtype=torch.float32,
62 + device=weight_scale.device)
63 +
64 + # Expand each scale to match the size of each logical matrix.
65 + start = 0
66 + for idx, logical_width in enumerate(logical_widths):
67 + end = start + logical_width
68 + weight_scale_channel[start:end, :] = weight_scale[idx]
69 + start = end
70 +
71 + return weight_scale_channel
72 +
73 +
74 +def requantize_with_max_scale(
75 + weight: torch.Tensor, weight_scale: torch.Tensor,
76 + logical_widths: List[int]) -> Tuple[torch.Tensor, torch.Tensor]:
77 + # Max scale to be used for requanitzation.
78 + max_w_scale = weight_scale.max()
79 +
80 + # QKV / MLP is fused in the on disk checkpoint if any of the
81 + # weight scales are still set to the default since we initialize
82 + # N weight scales for N shards but we only load 1 weight scale
83 + # from disk in this case. Skip requantization in this case (since)
84 + # we already are quantized with the single scale.
85 + # * Sample Model: nm-testing/Phi-3-mini-128k-instruct-FP8
86 + unfused_module_in_checkpoint = (weight_scale[-1] > torch.finfo(
87 + torch.float8_e4m3fn).min)
88 +
89 + # If unfused checkpoint, need requanize with the single scale.
90 + if unfused_module_in_checkpoint:
91 + start = 0
92 + for idx, logical_width in enumerate(logical_widths):
93 + end = start + logical_width
94 + weight_dq = per_tensor_dequantize(weight[start:end, :],
95 + weight_scale[idx])
96 + weight[start:end, :], _ = ops.scaled_fp8_quant(
97 + weight_dq, max_w_scale)
98 + start = end
99 +
100 + return max_w_scale, weight
101 +
102 +
103 +def apply_fp8_linear(
104 + input: torch.Tensor,
105 + weight: torch.Tensor,
106 + weight_scale: torch.Tensor,
107 + input_scale: torch.Tensor,
108 + bias: Optional[torch.Tensor] = None,
109 + cutlass_fp8_supported: bool = True,
110 +) -> torch.Tensor:
111 + # ops.scaled_fp8_quant supports both dynamic and static quant.
112 + # If dynamic, layer.input_scale is None and x_scale computed from x.
113 + # If static, layer.input_scale is scalar and x_scale is input_scale.
114 +
115 + if bias is None and cutlass_fp8_supported:
116 + qinput, x_scale = ops.scaled_fp8_quant(input, input_scale)
117 +
118 + # Fused GEMM_DQ
119 + output = ops.cutlass_scaled_mm(qinput,
120 + weight,
121 + out_dtype=input.dtype,
122 + scale_a=x_scale,
123 + scale_b=weight_scale)
124 +
125 + else:
126 + qinput, x_scale = ops.scaled_fp8_quant(input,
127 + input_scale,
128 + batch_dim_padding=17)
129 +
130 + # Fused GEMM_DQ -- note we padded the input above because
131 + # torch._scaled_mm is more performant for matrices with
132 + # batch dimension > 16. Note that this could change
133 + # in the future.
134 + output, _ = torch._scaled_mm(qinput,
135 + weight,
136 + out_dtype=input.dtype,
137 + scale_a=x_scale,
138 + scale_b=weight_scale,
139 + bias=bias)
140 +
141 + return torch.narrow(output, 0, 0, input.shape[0])
142 +
143 +
144 +def apply_int8_linear(
145 + input: torch.Tensor,
146 + weight: torch.Tensor,
147 + weight_scale: torch.Tensor,
148 + input_scale: torch.Tensor,
149 + bias: Optional[torch.Tensor] = None,
150 +):
151 + if bias is not None:
152 + raise NotImplementedError("W8A8 with int8 does not yet support bias.")
153 +
154 + # ops.scaled_int8_quant supports both dynamic and static quant.
155 + # * dynamic, layer.input_scale is None and x_scale computed from x.
156 + # * static, layer.input_scale is scalar and x_scale is input_scale.
157 + x_q, x_scale = ops.scaled_int8_quant(input, input_scale)
158 +
159 + return ops.cutlass_scaled_mm(x_q,
160 + weight,
161 + scale_a=x_scale,
162 + scale_b=weight_scale,
163 + out_dtype=input.dtype)