@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e837b624f2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Feature][Hardware][Amd] Add fp8 Linear Layer for Rocm (#7210)

Charlie Fu committed Aug 16, 2024 at 12:06 UTC e837b624f25efe5d05412e95e18ed07ced880272
7 files changed +164 -49
csrc/quantization/fp8/common.cu
+32 -17
@@ -9,6 +9,18 @@
9
10 #include "../../reduction_utils.cuh"
11
12 +#ifndef USE_ROCM
13 +using FP8_TYPE = c10::Float8_e4m3fn;
14 +C10_HOST_DEVICE constexpr auto FP8_E4M3_MAX =
15 + std::numeric_limits<FP8_TYPE>::max();
16 +#else
17 + #include "amd/hip_float8.h"
18 +using FP8_TYPE = c10::Float8_e4m3fnuz;
19 +// Using the default max value from pytorch (240.0) will cause accuracy
20 +// issue when running dynamic quantization. Here use 224.0f for rocm.
21 +constexpr auto FP8_E4M3_MAX = 224.0f;
22 +#endif
23 +
24 namespace vllm {
25
26 __device__ __forceinline__ float atomicMaxFloat(float* addr, float value) {
@@ -21,11 +33,9 @@ __device__ __forceinline__ float atomicMaxFloat(float* addr, float value) {
33 return old;
34 }
35
24 -#define FP8_E4M3_MAX std::numeric_limits<c10::Float8_e4m3fn>::max()
25 -
36 template <bool is_scale_inverted>
27 -__device__ __forceinline__ c10::Float8_e4m3fn scaled_fp8_conversion(
28 - float const val, float const scale) {
37 +__device__ __forceinline__ FP8_TYPE scaled_fp8_conversion(float const val,
38 + float const scale) {
39 float x = 0.0f;
40 if constexpr (is_scale_inverted) {
41 x = val * scale;
@@ -34,7 +44,13 @@ __device__ __forceinline__ c10::Float8_e4m3fn scaled_fp8_conversion(
44 }
45
46 float r = fmax(-FP8_E4M3_MAX, fmin(x, FP8_E4M3_MAX));
47 +#ifndef USE_ROCM
48 return static_cast<c10::Float8_e4m3fn>(r);
49 +#else
50 + // Use hardware cvt instruction for fp8 on rocm
51 + return c10::Float8_e4m3fnuz(hip_fp8(r).data,
52 + c10::Float8_e4m3fnuz::from_bits());
53 +#endif
54 }
55
56 // Compute the absolute maximum m of the input tensor and store
@@ -74,8 +90,7 @@ __global__ void segmented_max_reduction(float* __restrict__ scale,
90 // Finally, since cache[0] contains the maximum for this thread block,
91 // atomically write the max to the target location
92 if (threadIdx.x == 0) {
77 - atomicMaxFloat(scale,
78 - cache[0] / std::numeric_limits<c10::Float8_e4m3fn>::max());
93 + atomicMaxFloat(scale, cache[0] / FP8_E4M3_MAX);
94 }
95 }
96
@@ -88,10 +103,10 @@ struct __align__(8) vec4_t {
103 };
104
105 typedef struct __align__(4) {
91 - c10::Float8_e4m3fn x;
92 - c10::Float8_e4m3fn y;
93 - c10::Float8_e4m3fn z;
94 - c10::Float8_e4m3fn w;
106 + FP8_TYPE x;
107 + FP8_TYPE y;
108 + FP8_TYPE z;
109 + FP8_TYPE w;
110 }
111 float8x4_t;
112
@@ -124,7 +139,7 @@ __device__ float thread_max_vec(scalar_t const* __restrict__ input,
139 }
140
141 template <typename scalar_t, bool is_scale_inverted>
127 -__device__ void scaled_fp8_conversion_vec(c10::Float8_e4m3fn* __restrict__ out,
142 +__device__ void scaled_fp8_conversion_vec(FP8_TYPE* __restrict__ out,
143 scalar_t const* __restrict__ input,
144 float const scale,
145 int64_t const num_elems,
@@ -160,7 +175,7 @@ __device__ void scaled_fp8_conversion_vec(c10::Float8_e4m3fn* __restrict__ out,
175 }
176
177 template <typename scalar_t>
163 -__global__ void scaled_fp8_quant_kernel(c10::Float8_e4m3fn* __restrict__ out,
178 +__global__ void scaled_fp8_quant_kernel(FP8_TYPE* __restrict__ out,
179 const scalar_t* __restrict__ input,
180 const float* __restrict__ scale,
181 int64_t num_elems) {
@@ -175,7 +190,7 @@ __global__ void scaled_fp8_quant_kernel(c10::Float8_e4m3fn* __restrict__ out,
190
191 template <typename scalar_t>
192 __global__ void dynamic_per_token_scaled_fp8_quant_kernel(
178 - c10::Float8_e4m3fn* __restrict__ out, float* __restrict__ scale,
193 + FP8_TYPE* __restrict__ out, float* __restrict__ scale,
194 scalar_t const* __restrict__ input, float const* __restrict__ scale_ub,
195 const int hidden_size) {
196 float const min_scaling_factor = 1.0f / (FP8_E4M3_MAX * 512.f);
@@ -184,7 +199,7 @@ __global__ void dynamic_per_token_scaled_fp8_quant_kernel(
199 int const token_idx = blockIdx.x;
200
201 scalar_t const* __restrict__ token_input = &input[token_idx * hidden_size];
187 - c10::Float8_e4m3fn* __restrict__ token_output = &out[token_idx * hidden_size];
202 + FP8_TYPE* __restrict__ token_output = &out[token_idx * hidden_size];
203
204 // For vectorization, token_input and token_output pointers need to be
205 // aligned at 8-byte and 4-byte addresses respectively.
@@ -241,7 +256,7 @@ void static_scaled_fp8_quant(torch::Tensor& out, // [..., d]
256 VLLM_DISPATCH_FLOATING_TYPES(
257 input.scalar_type(), "scaled_fp8_quant_kernel", [&] {
258 vllm::scaled_fp8_quant_kernel<scalar_t><<<grid, block, 0, stream>>>(
244 - out.data_ptr<c10::Float8_e4m3fn>(), input.data_ptr<scalar_t>(),
259 + out.data_ptr<FP8_TYPE>(), input.data_ptr<scalar_t>(),
260 scale.data_ptr<float>(), num_elems);
261 });
262 }
@@ -261,7 +276,7 @@ void dynamic_scaled_fp8_quant(torch::Tensor& out, // [..., d]
276 vllm::segmented_max_reduction<scalar_t><<<grid, block, 0, stream>>>(
277 scale.data_ptr<float>(), input.data_ptr<scalar_t>(), num_elems);
278 vllm::scaled_fp8_quant_kernel<scalar_t><<<grid, block, 0, stream>>>(
264 - out.data_ptr<c10::Float8_e4m3fn>(), input.data_ptr<scalar_t>(),
279 + out.data_ptr<FP8_TYPE>(), input.data_ptr<scalar_t>(),
280 scale.data_ptr<float>(), num_elems);
281 });
282 }
@@ -284,7 +299,7 @@ void dynamic_per_token_scaled_fp8_quant(
299 input.scalar_type(), "dynamic_per_token_scaled_fp8_quant_kernel", [&] {
300 vllm::dynamic_per_token_scaled_fp8_quant_kernel<scalar_t>
301 <<<grid, block, 0, stream>>>(
287 - out.data_ptr<c10::Float8_e4m3fn>(), scales.data_ptr<float>(),
302 + out.data_ptr<FP8_TYPE>(), scales.data_ptr<float>(),
303 input.data_ptr<scalar_t>(),
304 scale_ub.has_value() ? scale_ub->data_ptr<float>() : nullptr,
305 hidden_size);
tests/kernels/quant_utils.py
+22 -11
@@ -2,6 +2,13 @@ from typing import Optional, Tuple, Union
2
3 import torch
4
5 +from vllm.utils import is_hip
6 +
7 +# Using the default value (240.0) from pytorch will cause accuracy
8 +# issue on dynamic quantization models. Here use 224.0 for rocm.
9 +ROCM_FP8_MAX = 224.0
10 +FP8_DTYPE = torch.float8_e4m3fnuz if is_hip() else torch.float8_e4m3fn
11 +
12
13 def as_float32_tensor(x: Union[float, torch.tensor]) -> torch.tensor:
14 return torch.as_tensor(x, dtype=torch.float32, device='cuda')
@@ -11,13 +18,15 @@ def ref_dynamic_per_token_quant(x: torch.tensor,
18 scale_ub: Optional[torch.tensor] = None) \
19 -> Tuple[torch.tensor, torch.tensor]:
20
14 - assert quant_dtype in [torch.int8, torch.float8_e4m3fn]
21 + assert quant_dtype in [torch.int8, FP8_DTYPE]
22 if scale_ub is not None:
16 - assert quant_dtype == torch.float8_e4m3fn
23 + assert quant_dtype == FP8_DTYPE
24
25 qtype_traits = torch.iinfo(quant_dtype) if quant_dtype == torch.int8 \
26 else torch.finfo(quant_dtype)
20 - qtype_max = as_float32_tensor(qtype_traits.max)
27 + qtype_traits_max = ROCM_FP8_MAX if is_hip() else qtype_traits.max
28 + qtype_traits_min = -ROCM_FP8_MAX if is_hip() else qtype_traits.min
29 + qtype_max = as_float32_tensor(qtype_traits_max)
30 s_1 = as_float32_tensor(1.0)
31 s_512 = as_float32_tensor(512.0)
32
@@ -37,15 +46,15 @@ def ref_dynamic_per_token_quant(x: torch.tensor,
46 iscales = as_float32_tensor(s_1 / scales)
47 torch_out = as_float32_tensor(x) * iscales
48 torch_out = torch_out.round()
40 - torch_out = torch_out.clamp(qtype_traits.min,
41 - qtype_traits.max).to(quant_dtype)
49 + torch_out = torch_out.clamp(qtype_traits_min,
50 + qtype_traits_max).to(quant_dtype)
51 else:
43 - assert quant_dtype == torch.float8_e4m3fn
52 + assert quant_dtype == FP8_DTYPE
53 min_scaling_factor = s_1 / (qtype_max * s_512)
54 scales = scales.clamp(min=min_scaling_factor)
55 torch_out = as_float32_tensor(x) / scales
47 - torch_out = torch_out.clamp(qtype_traits.min,
48 - qtype_traits.max).to(quant_dtype)
56 + torch_out = torch_out.clamp(qtype_traits_min,
57 + qtype_traits_max).to(quant_dtype)
58
59 return torch_out, scales
60
@@ -56,8 +65,10 @@ def ref_dynamic_per_token_quant(x: torch.tensor,
65 def ref_dynamic_per_tensor_fp8_quant(x: torch.tensor) \
66 -> Tuple[torch.tensor, torch.tensor]:
67
59 - fp8_traits = torch.finfo(torch.float8_e4m3fn)
60 - fp8_max = as_float32_tensor(fp8_traits.max)
68 + fp8_traits = torch.finfo(FP8_DTYPE)
69 + fp8_traits_max = ROCM_FP8_MAX if is_hip() else fp8_traits.max
70 + fp8_traits_min = -ROCM_FP8_MAX if is_hip() else fp8_traits.min
71 + fp8_max = as_float32_tensor(fp8_traits_max)
72 one = as_float32_tensor(1.0)
73
74 # For fp8, in order to match the cuda kernel output, we have to do exactly
@@ -68,5 +79,5 @@ def ref_dynamic_per_tensor_fp8_quant(x: torch.tensor) \
79 ref_scale = x_max / fp8_max
80 ref_iscale = one / ref_scale
81 ref_out = (as_float32_tensor(x) * ref_iscale).clamp(
71 - fp8_traits.min, fp8_traits.max).to(dtype=torch.float8_e4m3fn)
82 + fp8_traits_min, fp8_traits_max).to(FP8_DTYPE)
83 return ref_out, ref_scale.view((1, ))
tests/kernels/test_fp8_quant.py
+3 -3
@@ -2,7 +2,8 @@ import pytest
2 import torch
3
4 import vllm._custom_ops as ops
5 -from tests.kernels.quant_utils import (ref_dynamic_per_tensor_fp8_quant,
5 +from tests.kernels.quant_utils import (FP8_DTYPE,
6 + ref_dynamic_per_tensor_fp8_quant,
7 ref_dynamic_per_token_quant)
8
9 DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -31,8 +32,7 @@ def test_dynamic_per_token_fp8_quant(num_tokens: int, hidden_size: int,
32
33 scale_ub = torch.mean(x).to(dtype=torch.float32, device='cuda') \
34 if scale_ub else None
34 - ref_out, ref_scales = ref_dynamic_per_token_quant(x, torch.float8_e4m3fn,
35 - scale_ub)
35 + ref_out, ref_scales = ref_dynamic_per_token_quant(x, FP8_DTYPE, scale_ub)
36 ops_out, ops_scales = ops.scaled_fp8_quant(x,
37 scale_ub=scale_ub,
38 use_per_token_if_dynamic=True)
vllm/_custom_ops.py
+4 -1
@@ -369,9 +369,12 @@ def scaled_fp8_quant(
369 # This code assumes batch_dim and num_tokens are flattened
370 assert (input.ndim == 2)
371 shape: Union[Tuple[int, int], torch.Size] = input.shape
372 + # For rocm, the output fp8 dtype is torch.float_e3m3fnuz
373 + out_dtype: torch.dtype = torch.float8_e4m3fnuz if vllm.utils.is_hip() \
374 + else torch.float8_e4m3fn
375 if num_token_padding:
376 shape = (max(num_token_padding, input.shape[0]), shape[1])
374 - output = torch.empty(shape, device=input.device, dtype=torch.float8_e4m3fn)
377 + output = torch.empty(shape, device=input.device, dtype=out_dtype)
378
379 if scale is None:
380 if use_per_token_if_dynamic:
vllm/config.py
+1 -1
@@ -240,7 +240,7 @@ class ModelConfig:
240
241 def _verify_quantization(self) -> None:
242 supported_quantization = [*QUANTIZATION_METHODS]
243 - rocm_supported_quantization = ["gptq", "squeezellm"]
243 + rocm_supported_quantization = ["gptq", "squeezellm", "fp8"]
244 optimized_quantization_methods = [
245 "fp8", "marlin", "gptq_marlin_24", "gptq_marlin", "awq_marlin",
246 "fbgemm_fp8", "compressed_tensors", "compressed-tensors"
vllm/model_executor/layers/quantization/fp8.py
+55 -9
@@ -20,10 +20,11 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
20 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
21 all_close_1d, apply_fp8_linear, convert_to_channelwise,
22 create_per_tensor_scale_param, cutlass_fp8_supported,
23 - per_tensor_dequantize, requantize_with_max_scale)
23 + normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize,
24 + requantize_with_max_scale)
25 from vllm.model_executor.utils import set_weight_attrs
26 from vllm.platforms import current_platform
26 -from vllm.utils import print_warning_once
27 +from vllm.utils import is_hip, print_warning_once
28
29 ACTIVATION_SCHEMES = ["static", "dynamic"]
30
@@ -120,6 +121,9 @@ class Fp8LinearMethod(LinearMethodBase):
121 capability = current_platform.get_device_capability()
122 capability = capability[0] * 10 + capability[1]
123 self.use_marlin = capability < 89 or envs.VLLM_TEST_FORCE_FP8_MARLIN
124 + # Disable marlin for rocm
125 + if is_hip():
126 + self.use_marlin = False
127
128 def create_weights(
129 self,
@@ -168,6 +172,8 @@ class Fp8LinearMethod(LinearMethodBase):
172 scale = create_per_tensor_scale_param(output_partition_sizes,
173 **extra_weight_attrs)
174 layer.register_parameter("input_scale", scale)
175 + else:
176 + layer.register_parameter("input_scale", None)
177
178 def process_weights_after_loading(self, layer: Module) -> None:
179 # If checkpoint not serialized fp8, quantize the weights.
@@ -202,9 +208,23 @@ class Fp8LinearMethod(LinearMethodBase):
208 # requantize the logical shards as a single weight.
209 else:
210 # Dequant -> Quant with max scale so we can run per tensor.
211 + weight = layer.weight
212 + weight_scale = layer.weight_scale
213 +
214 + # If rocm, use float8_e4m3fnuz.
215 + if is_hip():
216 + weight, weight_scale, input_scale = \
217 + normalize_e4m3fn_to_e4m3fnuz(
218 + weight=weight,
219 + weight_scale=weight_scale,
220 + input_scale=layer.input_scale)
221 + if input_scale is not None:
222 + layer.input_scale = Parameter(input_scale,
223 + requires_grad=False)
224 +
225 weight_scale, weight = requantize_with_max_scale(
206 - weight=layer.weight,
207 - weight_scale=layer.weight_scale,
226 + weight=weight,
227 + weight_scale=weight_scale,
228 logical_widths=layer.logical_widths,
229 )
230
@@ -214,8 +234,6 @@ class Fp8LinearMethod(LinearMethodBase):
234 if self.quant_config.activation_scheme == "static":
235 layer.input_scale = Parameter(layer.input_scale.max(),
236 requires_grad=False)
217 - else:
218 - layer.input_scale = None
237
238 if self.use_marlin:
239 prepare_fp8_layer_for_marlin(layer)
@@ -346,10 +364,12 @@ class Fp8MoEMethod(FusedMoEMethodBase):
364
365 # If checkpoint is fp16, quantize in place.
366 if not self.quant_config.is_checkpoint_fp8_serialized:
367 + # If rocm, use float8_e4m3fnuz as dtype
368 + fp8_dtype = torch.float8_e4m3fnuz \
369 + if is_hip() else torch.float8_e4m3fn
370 w13_weight = torch.empty_like(layer.w13_weight.data,
350 - dtype=torch.float8_e4m3fn)
351 - w2_weight = torch.empty_like(layer.w2_weight.data,
352 - dtype=torch.float8_e4m3fn)
371 + dtype=fp8_dtype)
372 + w2_weight = torch.empty_like(layer.w2_weight.data, dtype=fp8_dtype)
373
374 # Re-initialize w13_scale because we directly quantize
375 # merged w13 weights and generate a single scaling factor.
@@ -393,6 +413,32 @@ class Fp8MoEMethod(FusedMoEMethodBase):
413 layer.w13_input_scale.max(), requires_grad=False)
414 layer.w2_input_scale = torch.nn.Parameter(
415 layer.w2_input_scale.max(), requires_grad=False)
416 + # If rocm, normalize the weights and scales to e4m3fnuz
417 + if is_hip():
418 + # Normalize the weights and scales
419 + w13_weight, w13_weight_scale, w13_input_scale = \
420 + normalize_e4m3fn_to_e4m3fnuz(
421 + layer.w13_weight, layer.w13_weight_scale,
422 + layer.w13_input_scale)
423 + w2_weight, w2_weight_scale, w2_input_scale = \
424 + normalize_e4m3fn_to_e4m3fnuz(
425 + layer.w2_weight, layer.w2_weight_scale,
426 + layer.w2_input_scale)
427 + # Reset the parameter
428 + layer.w13_weight = torch.nn.Parameter(w13_weight,
429 + requires_grad=False)
430 + layer.w13_weight_scale = torch.nn.Parameter(
431 + w13_weight_scale, requires_grad=False)
432 + if w13_input_scale is not None:
433 + layer.w13_input_scale = torch.nn.Parameter(
434 + w13_input_scale, requires_grad=False)
435 + layer.w2_weight = torch.nn.Parameter(w2_weight,
436 + requires_grad=False)
437 + layer.w2_weight_scale = torch.nn.Parameter(w2_weight_scale,
438 + requires_grad=False)
439 + if w2_input_scale is not None:
440 + layer.w2_input_scale = torch.nn.Parameter(
441 + w2_input_scale, requires_grad=False)
442
443 # Fp8 moe kernel needs single weight scale for w13 per expert.
444 # We take the max then dequant and requant each expert.
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+47 -7
@@ -6,9 +6,19 @@ from torch.nn import Parameter
6 from vllm import _custom_ops as ops
7 from vllm.model_executor.utils import set_weight_attrs
8 from vllm.platforms import current_platform
9 +from vllm.utils import is_hip
10 +
11 +# scaled_mm in pytorch on rocm has a bug that requires always
12 +# providing scaling factor for result. This value is created
13 +# as global value to avoid multiple tensor allocations, and
14 +# can be removed once pytorch fixes the bug.
15 +TORCH_SCALED_MM_SCALE_RESULT = torch.ones(1).cuda() if is_hip() else None
16
17
18 def cutlass_fp8_supported() -> bool:
19 + # cutlass is not supported on Rocm
20 + if is_hip():
21 + return False
22 capability = current_platform.get_device_capability()
23 capability = capability[0] * 10 + capability[1]
24
@@ -147,13 +157,19 @@ def apply_fp8_linear(
157
158 if per_tensor_weights and per_tensor_activations:
159 # Fused GEMM_DQ
150 - output, _ = torch._scaled_mm(qinput,
151 - weight,
152 - out_dtype=input.dtype,
153 - scale_a=x_scale,
154 - scale_b=weight_scale,
155 - bias=bias)
156 - return torch.narrow(output, 0, 0, input.shape[0])
160 + output = torch._scaled_mm(
161 + qinput,
162 + weight,
163 + out_dtype=input.dtype,
164 + scale_a=x_scale,
165 + scale_b=weight_scale,
166 + scale_result=TORCH_SCALED_MM_SCALE_RESULT,
167 + bias=bias)
168 + # Since in torch 2.5, scaled_mm only returns single value
169 + # This should be removed when vllm-nvidia also moves to 2.5
170 + if is_hip():
171 + return torch.narrow(output, 0, 0, input.shape[0])
172 + return torch.narrow(output[0], 0, 0, input.shape[0])
173
174 else:
175 # Fallback for channelwise case, where we use unfused DQ
@@ -207,3 +223,27 @@ def apply_int8_linear(
223 scale_b=weight_scale,
224 out_dtype=input.dtype,
225 bias=bias)
226 +
227 +
228 +def normalize_e4m3fn_to_e4m3fnuz(
229 + weight: torch.Tensor,
230 + weight_scale: torch.Tensor,
231 + input_scale: Optional[torch.Tensor] = None
232 +) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
233 + assert weight.dtype == torch.float8_e4m3fn
234 + # The bits pattern 10000000(-128) represents zero in e4m3fn
235 + # but NaN in e4m3fnuz. So here we set it to 0.
236 + # https://onnx.ai/onnx/technical/float8.html
237 + weight_as_int8 = weight.view(torch.int8)
238 + ROCM_FP8_NAN_AS_INT = -128
239 + weight_as_int8[weight_as_int8 == ROCM_FP8_NAN_AS_INT] = 0
240 + weight = weight_as_int8.view(torch.float8_e4m3fnuz)
241 +
242 + # For the same bits representation, e4m3fnuz value is half of
243 + # the e4m3fn value, so we should double the scaling factor to
244 + # get the same dequantized value.
245 + # https://onnx.ai/onnx/technical/float8.html
246 + weight_scale = weight_scale * 2.0
247 + if input_scale is not None:
248 + input_scale = input_scale * 2.0
249 + return weight, weight_scale, input_scale