A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[AMD][ROCm]Quantization methods on ROCm; Fix _scaled_mm call (#8380)
Co-authored-by: Alexei-V-Ivanov-AMD <156011006+Alexei-V-Ivanov-AMD@users.noreply.github.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>
Gregory Shtrasberg committed
Sep 18, 2024 at 13:41 UTC
b3195bc9e4d57b6107af2222afea26c51475e262
4 files changed
+71
-27
vllm/config.py
+4
-1
@@ -255,7 +255,10 @@ class ModelConfig:
255
256
def _verify_quantization(self) -> None:
257
supported_quantization = [*QUANTIZATION_METHODS]
258
- rocm_supported_quantization = ["awq", "gptq", "fp8"]
258
+ rocm_supported_quantization = [
259
+ "awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors",
260
+ "fbgemm_fp8"
261
+ ]
262
optimized_quantization_methods = [
263
"fp8", "marlin", "modelopt", "gptq_marlin_24", "gptq_marlin",
264
"awq_marlin", "fbgemm_fp8", "compressed_tensors",
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+26
-3
@@ -8,10 +8,12 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9
QuantizationStrategy)
10
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11
- apply_fp8_linear, cutlass_fp8_supported, requantize_with_max_scale)
11
+ apply_fp8_linear, cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz,
12
+ requantize_with_max_scale)
13
from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
14
ModelWeightParameter,
15
PerTensorScaleParameter)
16
+from vllm.utils import is_hip
17
18
__all__ = ["CompressedTensorsW8A8Fp8"]
19
@@ -39,16 +41,37 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
41
logical_widths=layer.logical_widths,
42
)
43
44
+ if is_hip():
45
+ weight, max_w_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
46
+ weight=weight,
47
+ weight_scale=max_w_scale,
48
+ input_scale=layer.input_scale)
49
+ if input_scale is not None:
50
+ layer.input_scale = Parameter(input_scale,
51
+ requires_grad=False)
52
+
53
layer.weight = Parameter(weight.t(), requires_grad=False)
54
layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
55
56
# If channelwise, scales are already lined up, so just transpose.
57
elif self.strategy == QuantizationStrategy.CHANNEL:
58
weight = layer.weight
59
+
60
+ if is_hip():
61
+ weight, weight_scale, input_scale = \
62
+ normalize_e4m3fn_to_e4m3fnuz(
63
+ weight=weight,
64
+ weight_scale=layer.weight_scale,
65
+ input_scale=layer.input_scale)
66
+ if input_scale is not None:
67
+ layer.input_scale = Parameter(input_scale,
68
+ requires_grad=False)
69
+ else:
70
+ weight_scale = layer.weight_scale.data
71
+
72
layer.weight = Parameter(weight.t(), requires_grad=False)
73
# required by torch.compile to be torch.nn.Parameter
50
- layer.weight_scale = Parameter(layer.weight_scale.data,
51
- requires_grad=False)
74
+ layer.weight_scale = Parameter(weight_scale, requires_grad=False)
75
76
else:
77
raise ValueError(f"Unknown quantization strategy {self.strategy}")
vllm/model_executor/layers/quantization/fbgemm_fp8.py
+13
-2
@@ -15,10 +15,11 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
15
from vllm.model_executor.layers.quantization.utils.quant_utils import (
16
is_layer_skipped)
17
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
18
- apply_fp8_linear)
18
+ apply_fp8_linear, normalize_e4m3fn_to_e4m3fnuz)
19
from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
20
ModelWeightParameter)
21
from vllm.platforms import current_platform
22
+from vllm.utils import is_hip
23
24
logger = init_logger(__name__)
25
@@ -125,8 +126,18 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
126
layer.weight = Parameter(layer.weight.data, requires_grad=False)
127
128
weight = layer.weight
128
- layer.weight = Parameter(weight.t(), requires_grad=False)
129
130
+ if is_hip():
131
+ weight, weight_scale, input_scale = \
132
+ normalize_e4m3fn_to_e4m3fnuz(
133
+ weight=weight,
134
+ weight_scale=layer.weight_scale,
135
+ input_scale=None)
136
+ if input_scale is not None:
137
+ layer.input_scale = Parameter(input_scale, requires_grad=False)
138
+ layer.weight_scale = Parameter(weight_scale, requires_grad=False)
139
+
140
+ layer.weight = Parameter(weight.t(), requires_grad=False)
141
if self.quant_config.use_marlin:
142
prepare_fp8_layer_for_marlin(layer)
143
# Activations not quantized for marlin.
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+28
-21
@@ -6,11 +6,9 @@ from vllm import _custom_ops as ops
6
from vllm.platforms import current_platform
7
from vllm.utils import is_hip
8
9
-# scaled_mm in pytorch on rocm has a bug that requires always
10
-# providing scaling factor for result. This value is created
11
-# as global value to avoid multiple tensor allocations, and
12
-# can be removed once pytorch fixes the bug.
13
-TORCH_SCALED_MM_SCALE_RESULT = torch.ones(1).cuda() if is_hip() else None
9
+# Input scaling factors are no longer optional in _scaled_mm starting
10
+# from pytorch 2.5. Allocating a dummy tensor to pass as input_scale
11
+TORCH_DEVICE_IDENTITY = torch.ones(1).cuda() if is_hip() else None
12
13
14
def cutlass_fp8_supported() -> bool:
@@ -131,19 +129,17 @@ def apply_fp8_linear(
129
130
if per_tensor_weights and per_tensor_activations:
131
# Fused GEMM_DQ
134
- output = torch._scaled_mm(
135
- qinput,
136
- weight,
137
- out_dtype=input.dtype,
138
- scale_a=x_scale,
139
- scale_b=weight_scale,
140
- scale_result=TORCH_SCALED_MM_SCALE_RESULT,
141
- bias=bias)
142
- # Since in torch 2.5, scaled_mm only returns single value
143
- # This should be removed when vllm-nvidia also moves to 2.5
144
- if is_hip():
145
- return torch.narrow(output, 0, 0, input.shape[0])
146
- return torch.narrow(output[0], 0, 0, input.shape[0])
132
+ output = torch._scaled_mm(qinput,
133
+ weight,
134
+ out_dtype=input.dtype,
135
+ scale_a=x_scale,
136
+ scale_b=weight_scale,
137
+ bias=bias)
138
+ # A fix for discrepancy in scaled_mm which returns tuple
139
+ # for torch < 2.5 and a single value in torch >= 2.5
140
+ if type(output) is tuple and len(output) == 2:
141
+ return torch.narrow(output[0], 0, 0, input.shape[0])
142
+ return torch.narrow(output, 0, 0, input.shape[0])
143
144
else:
145
# Fallback for channelwise case, where we use unfused DQ
@@ -161,12 +157,23 @@ def apply_fp8_linear(
157
# For the scaled_mm fallback case, we break this down, since it
158
# does not support s_w being a vector.
159
160
+ # Making sure the dummy tensor is on the same device as the weight
161
+ global TORCH_DEVICE_IDENTITY
162
+ if TORCH_DEVICE_IDENTITY.device != weight.device:
163
+ TORCH_DEVICE_IDENTITY = TORCH_DEVICE_IDENTITY.to(weight.device)
164
+
165
# GEMM
166
# This computes C = (X * W).
167
# Output in fp32 to allow subsequent ops to happen in-place
167
- output, _ = torch._scaled_mm(qinput,
168
- weight,
169
- out_dtype=torch.float32)
168
+ output = torch._scaled_mm(qinput,
169
+ weight,
170
+ scale_a=TORCH_DEVICE_IDENTITY,
171
+ scale_b=TORCH_DEVICE_IDENTITY,
172
+ out_dtype=torch.float32)
173
+ # A fix for discrepancy in scaled_mm which returns tuple
174
+ # for torch < 2.5 and a single value in torch >= 2.5
175
+ if type(output) is tuple and len(output) == 2:
176
+ output = output[0]
177
# Unpad (undo num_token_padding)
178
output = torch.narrow(output, 0, 0, input.shape[0])
179
x_scale = torch.narrow(x_scale, 0, 0, input.shape[0])