@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b3195bc9e4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[AMD][ROCm]Quantization methods on ROCm; Fix _scaled_mm call (#8380)

Co-authored-by: Alexei-V-Ivanov-AMD <156011006+Alexei-V-Ivanov-AMD@users.noreply.github.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>

Gregory Shtrasberg committed Sep 18, 2024 at 13:41 UTC b3195bc9e4d57b6107af2222afea26c51475e262
4 files changed +71 -27
vllm/config.py
+4 -1
@@ -255,7 +255,10 @@ class ModelConfig:
255
256 def _verify_quantization(self) -> None:
257 supported_quantization = [*QUANTIZATION_METHODS]
258 - rocm_supported_quantization = ["awq", "gptq", "fp8"]
258 + rocm_supported_quantization = [
259 + "awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors",
260 + "fbgemm_fp8"
261 + ]
262 optimized_quantization_methods = [
263 "fp8", "marlin", "modelopt", "gptq_marlin_24", "gptq_marlin",
264 "awq_marlin", "fbgemm_fp8", "compressed_tensors",
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+26 -3
@@ -8,10 +8,12 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9 QuantizationStrategy)
10 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11 - apply_fp8_linear, cutlass_fp8_supported, requantize_with_max_scale)
11 + apply_fp8_linear, cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz,
12 + requantize_with_max_scale)
13 from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
14 ModelWeightParameter,
15 PerTensorScaleParameter)
16 +from vllm.utils import is_hip
17
18 __all__ = ["CompressedTensorsW8A8Fp8"]
19
@@ -39,16 +41,37 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
41 logical_widths=layer.logical_widths,
42 )
43
44 + if is_hip():
45 + weight, max_w_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
46 + weight=weight,
47 + weight_scale=max_w_scale,
48 + input_scale=layer.input_scale)
49 + if input_scale is not None:
50 + layer.input_scale = Parameter(input_scale,
51 + requires_grad=False)
52 +
53 layer.weight = Parameter(weight.t(), requires_grad=False)
54 layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
55
56 # If channelwise, scales are already lined up, so just transpose.
57 elif self.strategy == QuantizationStrategy.CHANNEL:
58 weight = layer.weight
59 +
60 + if is_hip():
61 + weight, weight_scale, input_scale = \
62 + normalize_e4m3fn_to_e4m3fnuz(
63 + weight=weight,
64 + weight_scale=layer.weight_scale,
65 + input_scale=layer.input_scale)
66 + if input_scale is not None:
67 + layer.input_scale = Parameter(input_scale,
68 + requires_grad=False)
69 + else:
70 + weight_scale = layer.weight_scale.data
71 +
72 layer.weight = Parameter(weight.t(), requires_grad=False)
73 # required by torch.compile to be torch.nn.Parameter
50 - layer.weight_scale = Parameter(layer.weight_scale.data,
51 - requires_grad=False)
74 + layer.weight_scale = Parameter(weight_scale, requires_grad=False)
75
76 else:
77 raise ValueError(f"Unknown quantization strategy {self.strategy}")
vllm/model_executor/layers/quantization/fbgemm_fp8.py
+13 -2
@@ -15,10 +15,11 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
15 from vllm.model_executor.layers.quantization.utils.quant_utils import (
16 is_layer_skipped)
17 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
18 - apply_fp8_linear)
18 + apply_fp8_linear, normalize_e4m3fn_to_e4m3fnuz)
19 from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
20 ModelWeightParameter)
21 from vllm.platforms import current_platform
22 +from vllm.utils import is_hip
23
24 logger = init_logger(__name__)
25
@@ -125,8 +126,18 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
126 layer.weight = Parameter(layer.weight.data, requires_grad=False)
127
128 weight = layer.weight
128 - layer.weight = Parameter(weight.t(), requires_grad=False)
129
130 + if is_hip():
131 + weight, weight_scale, input_scale = \
132 + normalize_e4m3fn_to_e4m3fnuz(
133 + weight=weight,
134 + weight_scale=layer.weight_scale,
135 + input_scale=None)
136 + if input_scale is not None:
137 + layer.input_scale = Parameter(input_scale, requires_grad=False)
138 + layer.weight_scale = Parameter(weight_scale, requires_grad=False)
139 +
140 + layer.weight = Parameter(weight.t(), requires_grad=False)
141 if self.quant_config.use_marlin:
142 prepare_fp8_layer_for_marlin(layer)
143 # Activations not quantized for marlin.
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+28 -21
@@ -6,11 +6,9 @@ from vllm import _custom_ops as ops
6 from vllm.platforms import current_platform
7 from vllm.utils import is_hip
8
9 -# scaled_mm in pytorch on rocm has a bug that requires always
10 -# providing scaling factor for result. This value is created
11 -# as global value to avoid multiple tensor allocations, and
12 -# can be removed once pytorch fixes the bug.
13 -TORCH_SCALED_MM_SCALE_RESULT = torch.ones(1).cuda() if is_hip() else None
9 +# Input scaling factors are no longer optional in _scaled_mm starting
10 +# from pytorch 2.5. Allocating a dummy tensor to pass as input_scale
11 +TORCH_DEVICE_IDENTITY = torch.ones(1).cuda() if is_hip() else None
12
13
14 def cutlass_fp8_supported() -> bool:
@@ -131,19 +129,17 @@ def apply_fp8_linear(
129
130 if per_tensor_weights and per_tensor_activations:
131 # Fused GEMM_DQ
134 - output = torch._scaled_mm(
135 - qinput,
136 - weight,
137 - out_dtype=input.dtype,
138 - scale_a=x_scale,
139 - scale_b=weight_scale,
140 - scale_result=TORCH_SCALED_MM_SCALE_RESULT,
141 - bias=bias)
142 - # Since in torch 2.5, scaled_mm only returns single value
143 - # This should be removed when vllm-nvidia also moves to 2.5
144 - if is_hip():
145 - return torch.narrow(output, 0, 0, input.shape[0])
146 - return torch.narrow(output[0], 0, 0, input.shape[0])
132 + output = torch._scaled_mm(qinput,
133 + weight,
134 + out_dtype=input.dtype,
135 + scale_a=x_scale,
136 + scale_b=weight_scale,
137 + bias=bias)
138 + # A fix for discrepancy in scaled_mm which returns tuple
139 + # for torch < 2.5 and a single value in torch >= 2.5
140 + if type(output) is tuple and len(output) == 2:
141 + return torch.narrow(output[0], 0, 0, input.shape[0])
142 + return torch.narrow(output, 0, 0, input.shape[0])
143
144 else:
145 # Fallback for channelwise case, where we use unfused DQ
@@ -161,12 +157,23 @@ def apply_fp8_linear(
157 # For the scaled_mm fallback case, we break this down, since it
158 # does not support s_w being a vector.
159
160 + # Making sure the dummy tensor is on the same device as the weight
161 + global TORCH_DEVICE_IDENTITY
162 + if TORCH_DEVICE_IDENTITY.device != weight.device:
163 + TORCH_DEVICE_IDENTITY = TORCH_DEVICE_IDENTITY.to(weight.device)
164 +
165 # GEMM
166 # This computes C = (X * W).
167 # Output in fp32 to allow subsequent ops to happen in-place
167 - output, _ = torch._scaled_mm(qinput,
168 - weight,
169 - out_dtype=torch.float32)
168 + output = torch._scaled_mm(qinput,
169 + weight,
170 + scale_a=TORCH_DEVICE_IDENTITY,
171 + scale_b=TORCH_DEVICE_IDENTITY,
172 + out_dtype=torch.float32)
173 + # A fix for discrepancy in scaled_mm which returns tuple
174 + # for torch < 2.5 and a single value in torch >= 2.5
175 + if type(output) is tuple and len(output) == 2:
176 + output = output[0]
177 # Unpad (undo num_token_padding)
178 output = torch.narrow(output, 0, 0, input.shape[0])
179 x_scale = torch.narrow(x_scale, 0, 0, input.shape[0])