A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Feature][Hardware][Amd] Add fp8 Linear Layer for Rocm (#7210)
Charlie Fu committed
Aug 16, 2024 at 12:06 UTC
e837b624f25efe5d05412e95e18ed07ced880272
7 files changed
+164
-49
csrc/quantization/fp8/common.cu
+32
-17
@@ -9,6 +9,18 @@
9
10
#include "../../reduction_utils.cuh"
11
12
+#ifndef USE_ROCM
13
+using FP8_TYPE = c10::Float8_e4m3fn;
14
+C10_HOST_DEVICE constexpr auto FP8_E4M3_MAX =
15
+ std::numeric_limits<FP8_TYPE>::max();
16
+#else
17
+ #include "amd/hip_float8.h"
18
+using FP8_TYPE = c10::Float8_e4m3fnuz;
19
+// Using the default max value from pytorch (240.0) will cause accuracy
20
+// issue when running dynamic quantization. Here use 224.0f for rocm.
21
+constexpr auto FP8_E4M3_MAX = 224.0f;
22
+#endif
23
+
24
namespace vllm {
25
26
__device__ __forceinline__ float atomicMaxFloat(float* addr, float value) {
@@ -21,11 +33,9 @@ __device__ __forceinline__ float atomicMaxFloat(float* addr, float value) {
33
return old;
34
}
35
24
-#define FP8_E4M3_MAX std::numeric_limits<c10::Float8_e4m3fn>::max()
25
-
36
template <bool is_scale_inverted>
27
-__device__ __forceinline__ c10::Float8_e4m3fn scaled_fp8_conversion(
28
- float const val, float const scale) {
37
+__device__ __forceinline__ FP8_TYPE scaled_fp8_conversion(float const val,
38
+ float const scale) {
39
float x = 0.0f;
40
if constexpr (is_scale_inverted) {
41
x = val * scale;
@@ -34,7 +44,13 @@ __device__ __forceinline__ c10::Float8_e4m3fn scaled_fp8_conversion(
44
}
45
46
float r = fmax(-FP8_E4M3_MAX, fmin(x, FP8_E4M3_MAX));
47
+#ifndef USE_ROCM
48
return static_cast<c10::Float8_e4m3fn>(r);
49
+#else
50
+ // Use hardware cvt instruction for fp8 on rocm
51
+ return c10::Float8_e4m3fnuz(hip_fp8(r).data,
52
+ c10::Float8_e4m3fnuz::from_bits());
53
+#endif
54
}
55
56
// Compute the absolute maximum m of the input tensor and store
@@ -74,8 +90,7 @@ __global__ void segmented_max_reduction(float* __restrict__ scale,
90
// Finally, since cache[0] contains the maximum for this thread block,
91
// atomically write the max to the target location
92
if (threadIdx.x == 0) {
77
- atomicMaxFloat(scale,
78
- cache[0] / std::numeric_limits<c10::Float8_e4m3fn>::max());
93
+ atomicMaxFloat(scale, cache[0] / FP8_E4M3_MAX);
94
}
95
}
96
@@ -88,10 +103,10 @@ struct __align__(8) vec4_t {
103
};
104
105
typedef struct __align__(4) {
91
- c10::Float8_e4m3fn x;
92
- c10::Float8_e4m3fn y;
93
- c10::Float8_e4m3fn z;
94
- c10::Float8_e4m3fn w;
106
+ FP8_TYPE x;
107
+ FP8_TYPE y;
108
+ FP8_TYPE z;
109
+ FP8_TYPE w;
110
}
111
float8x4_t;
112
@@ -124,7 +139,7 @@ __device__ float thread_max_vec(scalar_t const* __restrict__ input,
139
}
140
141
template <typename scalar_t, bool is_scale_inverted>
127
-__device__ void scaled_fp8_conversion_vec(c10::Float8_e4m3fn* __restrict__ out,
142
+__device__ void scaled_fp8_conversion_vec(FP8_TYPE* __restrict__ out,
143
scalar_t const* __restrict__ input,
144
float const scale,
145
int64_t const num_elems,
@@ -160,7 +175,7 @@ __device__ void scaled_fp8_conversion_vec(c10::Float8_e4m3fn* __restrict__ out,
175
}
176
177
template <typename scalar_t>
163
-__global__ void scaled_fp8_quant_kernel(c10::Float8_e4m3fn* __restrict__ out,
178
+__global__ void scaled_fp8_quant_kernel(FP8_TYPE* __restrict__ out,
179
const scalar_t* __restrict__ input,
180
const float* __restrict__ scale,
181
int64_t num_elems) {
@@ -175,7 +190,7 @@ __global__ void scaled_fp8_quant_kernel(c10::Float8_e4m3fn* __restrict__ out,
190
191
template <typename scalar_t>
192
__global__ void dynamic_per_token_scaled_fp8_quant_kernel(
178
- c10::Float8_e4m3fn* __restrict__ out, float* __restrict__ scale,
193
+ FP8_TYPE* __restrict__ out, float* __restrict__ scale,
194
scalar_t const* __restrict__ input, float const* __restrict__ scale_ub,
195
const int hidden_size) {
196
float const min_scaling_factor = 1.0f / (FP8_E4M3_MAX * 512.f);
@@ -184,7 +199,7 @@ __global__ void dynamic_per_token_scaled_fp8_quant_kernel(
199
int const token_idx = blockIdx.x;
200
201
scalar_t const* __restrict__ token_input = &input[token_idx * hidden_size];
187
- c10::Float8_e4m3fn* __restrict__ token_output = &out[token_idx * hidden_size];
202
+ FP8_TYPE* __restrict__ token_output = &out[token_idx * hidden_size];
203
204
// For vectorization, token_input and token_output pointers need to be
205
// aligned at 8-byte and 4-byte addresses respectively.
@@ -241,7 +256,7 @@ void static_scaled_fp8_quant(torch::Tensor& out, // [..., d]
256
VLLM_DISPATCH_FLOATING_TYPES(
257
input.scalar_type(), "scaled_fp8_quant_kernel", [&] {
258
vllm::scaled_fp8_quant_kernel<scalar_t><<<grid, block, 0, stream>>>(
244
- out.data_ptr<c10::Float8_e4m3fn>(), input.data_ptr<scalar_t>(),
259
+ out.data_ptr<FP8_TYPE>(), input.data_ptr<scalar_t>(),
260
scale.data_ptr<float>(), num_elems);
261
});
262
}
@@ -261,7 +276,7 @@ void dynamic_scaled_fp8_quant(torch::Tensor& out, // [..., d]
276
vllm::segmented_max_reduction<scalar_t><<<grid, block, 0, stream>>>(
277
scale.data_ptr<float>(), input.data_ptr<scalar_t>(), num_elems);
278
vllm::scaled_fp8_quant_kernel<scalar_t><<<grid, block, 0, stream>>>(
264
- out.data_ptr<c10::Float8_e4m3fn>(), input.data_ptr<scalar_t>(),
279
+ out.data_ptr<FP8_TYPE>(), input.data_ptr<scalar_t>(),
280
scale.data_ptr<float>(), num_elems);
281
});
282
}
@@ -284,7 +299,7 @@ void dynamic_per_token_scaled_fp8_quant(
299
input.scalar_type(), "dynamic_per_token_scaled_fp8_quant_kernel", [&] {
300
vllm::dynamic_per_token_scaled_fp8_quant_kernel<scalar_t>
301
<<<grid, block, 0, stream>>>(
287
- out.data_ptr<c10::Float8_e4m3fn>(), scales.data_ptr<float>(),
302
+ out.data_ptr<FP8_TYPE>(), scales.data_ptr<float>(),
303
input.data_ptr<scalar_t>(),
304
scale_ub.has_value() ? scale_ub->data_ptr<float>() : nullptr,
305
hidden_size);
tests/kernels/quant_utils.py
+22
-11
@@ -2,6 +2,13 @@ from typing import Optional, Tuple, Union
2
3
import torch
4
5
+from vllm.utils import is_hip
6
+
7
+# Using the default value (240.0) from pytorch will cause accuracy
8
+# issue on dynamic quantization models. Here use 224.0 for rocm.
9
+ROCM_FP8_MAX = 224.0
10
+FP8_DTYPE = torch.float8_e4m3fnuz if is_hip() else torch.float8_e4m3fn
11
+
12
13
def as_float32_tensor(x: Union[float, torch.tensor]) -> torch.tensor:
14
return torch.as_tensor(x, dtype=torch.float32, device='cuda')
@@ -11,13 +18,15 @@ def ref_dynamic_per_token_quant(x: torch.tensor,
18
scale_ub: Optional[torch.tensor] = None) \
19
-> Tuple[torch.tensor, torch.tensor]:
20
14
- assert quant_dtype in [torch.int8, torch.float8_e4m3fn]
21
+ assert quant_dtype in [torch.int8, FP8_DTYPE]
22
if scale_ub is not None:
16
- assert quant_dtype == torch.float8_e4m3fn
23
+ assert quant_dtype == FP8_DTYPE
24
25
qtype_traits = torch.iinfo(quant_dtype) if quant_dtype == torch.int8 \
26
else torch.finfo(quant_dtype)
20
- qtype_max = as_float32_tensor(qtype_traits.max)
27
+ qtype_traits_max = ROCM_FP8_MAX if is_hip() else qtype_traits.max
28
+ qtype_traits_min = -ROCM_FP8_MAX if is_hip() else qtype_traits.min
29
+ qtype_max = as_float32_tensor(qtype_traits_max)
30
s_1 = as_float32_tensor(1.0)
31
s_512 = as_float32_tensor(512.0)
32
@@ -37,15 +46,15 @@ def ref_dynamic_per_token_quant(x: torch.tensor,
46
iscales = as_float32_tensor(s_1 / scales)
47
torch_out = as_float32_tensor(x) * iscales
48
torch_out = torch_out.round()
40
- torch_out = torch_out.clamp(qtype_traits.min,
41
- qtype_traits.max).to(quant_dtype)
49
+ torch_out = torch_out.clamp(qtype_traits_min,
50
+ qtype_traits_max).to(quant_dtype)
51
else:
43
- assert quant_dtype == torch.float8_e4m3fn
52
+ assert quant_dtype == FP8_DTYPE
53
min_scaling_factor = s_1 / (qtype_max * s_512)
54
scales = scales.clamp(min=min_scaling_factor)
55
torch_out = as_float32_tensor(x) / scales
47
- torch_out = torch_out.clamp(qtype_traits.min,
48
- qtype_traits.max).to(quant_dtype)
56
+ torch_out = torch_out.clamp(qtype_traits_min,
57
+ qtype_traits_max).to(quant_dtype)
58
59
return torch_out, scales
60
@@ -56,8 +65,10 @@ def ref_dynamic_per_token_quant(x: torch.tensor,
65
def ref_dynamic_per_tensor_fp8_quant(x: torch.tensor) \
66
-> Tuple[torch.tensor, torch.tensor]:
67
59
- fp8_traits = torch.finfo(torch.float8_e4m3fn)
60
- fp8_max = as_float32_tensor(fp8_traits.max)
68
+ fp8_traits = torch.finfo(FP8_DTYPE)
69
+ fp8_traits_max = ROCM_FP8_MAX if is_hip() else fp8_traits.max
70
+ fp8_traits_min = -ROCM_FP8_MAX if is_hip() else fp8_traits.min
71
+ fp8_max = as_float32_tensor(fp8_traits_max)
72
one = as_float32_tensor(1.0)
73
74
# For fp8, in order to match the cuda kernel output, we have to do exactly
@@ -68,5 +79,5 @@ def ref_dynamic_per_tensor_fp8_quant(x: torch.tensor) \
79
ref_scale = x_max / fp8_max
80
ref_iscale = one / ref_scale
81
ref_out = (as_float32_tensor(x) * ref_iscale).clamp(
71
- fp8_traits.min, fp8_traits.max).to(dtype=torch.float8_e4m3fn)
82
+ fp8_traits_min, fp8_traits_max).to(FP8_DTYPE)
83
return ref_out, ref_scale.view((1, ))
tests/kernels/test_fp8_quant.py
+3
-3
@@ -2,7 +2,8 @@ import pytest
2
import torch
3
4
import vllm._custom_ops as ops
5
-from tests.kernels.quant_utils import (ref_dynamic_per_tensor_fp8_quant,
5
+from tests.kernels.quant_utils import (FP8_DTYPE,
6
+ ref_dynamic_per_tensor_fp8_quant,
7
ref_dynamic_per_token_quant)
8
9
DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -31,8 +32,7 @@ def test_dynamic_per_token_fp8_quant(num_tokens: int, hidden_size: int,
32
33
scale_ub = torch.mean(x).to(dtype=torch.float32, device='cuda') \
34
if scale_ub else None
34
- ref_out, ref_scales = ref_dynamic_per_token_quant(x, torch.float8_e4m3fn,
35
- scale_ub)
35
+ ref_out, ref_scales = ref_dynamic_per_token_quant(x, FP8_DTYPE, scale_ub)
36
ops_out, ops_scales = ops.scaled_fp8_quant(x,
37
scale_ub=scale_ub,
38
use_per_token_if_dynamic=True)
vllm/_custom_ops.py
+4
-1
@@ -369,9 +369,12 @@ def scaled_fp8_quant(
369
# This code assumes batch_dim and num_tokens are flattened
370
assert (input.ndim == 2)
371
shape: Union[Tuple[int, int], torch.Size] = input.shape
372
+ # For rocm, the output fp8 dtype is torch.float_e3m3fnuz
373
+ out_dtype: torch.dtype = torch.float8_e4m3fnuz if vllm.utils.is_hip() \
374
+ else torch.float8_e4m3fn
375
if num_token_padding:
376
shape = (max(num_token_padding, input.shape[0]), shape[1])
374
- output = torch.empty(shape, device=input.device, dtype=torch.float8_e4m3fn)
377
+ output = torch.empty(shape, device=input.device, dtype=out_dtype)
378
379
if scale is None:
380
if use_per_token_if_dynamic:
vllm/config.py
+1
-1
@@ -240,7 +240,7 @@ class ModelConfig:
240
241
def _verify_quantization(self) -> None:
242
supported_quantization = [*QUANTIZATION_METHODS]
243
- rocm_supported_quantization = ["gptq", "squeezellm"]
243
+ rocm_supported_quantization = ["gptq", "squeezellm", "fp8"]
244
optimized_quantization_methods = [
245
"fp8", "marlin", "gptq_marlin_24", "gptq_marlin", "awq_marlin",
246
"fbgemm_fp8", "compressed_tensors", "compressed-tensors"
vllm/model_executor/layers/quantization/fp8.py
+55
-9
@@ -20,10 +20,11 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
20
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
21
all_close_1d, apply_fp8_linear, convert_to_channelwise,
22
create_per_tensor_scale_param, cutlass_fp8_supported,
23
- per_tensor_dequantize, requantize_with_max_scale)
23
+ normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize,
24
+ requantize_with_max_scale)
25
from vllm.model_executor.utils import set_weight_attrs
26
from vllm.platforms import current_platform
26
-from vllm.utils import print_warning_once
27
+from vllm.utils import is_hip, print_warning_once
28
29
ACTIVATION_SCHEMES = ["static", "dynamic"]
30
@@ -120,6 +121,9 @@ class Fp8LinearMethod(LinearMethodBase):
121
capability = current_platform.get_device_capability()
122
capability = capability[0] * 10 + capability[1]
123
self.use_marlin = capability < 89 or envs.VLLM_TEST_FORCE_FP8_MARLIN
124
+ # Disable marlin for rocm
125
+ if is_hip():
126
+ self.use_marlin = False
127
128
def create_weights(
129
self,
@@ -168,6 +172,8 @@ class Fp8LinearMethod(LinearMethodBase):
172
scale = create_per_tensor_scale_param(output_partition_sizes,
173
**extra_weight_attrs)
174
layer.register_parameter("input_scale", scale)
175
+ else:
176
+ layer.register_parameter("input_scale", None)
177
178
def process_weights_after_loading(self, layer: Module) -> None:
179
# If checkpoint not serialized fp8, quantize the weights.
@@ -202,9 +208,23 @@ class Fp8LinearMethod(LinearMethodBase):
208
# requantize the logical shards as a single weight.
209
else:
210
# Dequant -> Quant with max scale so we can run per tensor.
211
+ weight = layer.weight
212
+ weight_scale = layer.weight_scale
213
+
214
+ # If rocm, use float8_e4m3fnuz.
215
+ if is_hip():
216
+ weight, weight_scale, input_scale = \
217
+ normalize_e4m3fn_to_e4m3fnuz(
218
+ weight=weight,
219
+ weight_scale=weight_scale,
220
+ input_scale=layer.input_scale)
221
+ if input_scale is not None:
222
+ layer.input_scale = Parameter(input_scale,
223
+ requires_grad=False)
224
+
225
weight_scale, weight = requantize_with_max_scale(
206
- weight=layer.weight,
207
- weight_scale=layer.weight_scale,
226
+ weight=weight,
227
+ weight_scale=weight_scale,
228
logical_widths=layer.logical_widths,
229
)
230
@@ -214,8 +234,6 @@ class Fp8LinearMethod(LinearMethodBase):
234
if self.quant_config.activation_scheme == "static":
235
layer.input_scale = Parameter(layer.input_scale.max(),
236
requires_grad=False)
217
- else:
218
- layer.input_scale = None
237
238
if self.use_marlin:
239
prepare_fp8_layer_for_marlin(layer)
@@ -346,10 +364,12 @@ class Fp8MoEMethod(FusedMoEMethodBase):
364
365
# If checkpoint is fp16, quantize in place.
366
if not self.quant_config.is_checkpoint_fp8_serialized:
367
+ # If rocm, use float8_e4m3fnuz as dtype
368
+ fp8_dtype = torch.float8_e4m3fnuz \
369
+ if is_hip() else torch.float8_e4m3fn
370
w13_weight = torch.empty_like(layer.w13_weight.data,
350
- dtype=torch.float8_e4m3fn)
351
- w2_weight = torch.empty_like(layer.w2_weight.data,
352
- dtype=torch.float8_e4m3fn)
371
+ dtype=fp8_dtype)
372
+ w2_weight = torch.empty_like(layer.w2_weight.data, dtype=fp8_dtype)
373
374
# Re-initialize w13_scale because we directly quantize
375
# merged w13 weights and generate a single scaling factor.
@@ -393,6 +413,32 @@ class Fp8MoEMethod(FusedMoEMethodBase):
413
layer.w13_input_scale.max(), requires_grad=False)
414
layer.w2_input_scale = torch.nn.Parameter(
415
layer.w2_input_scale.max(), requires_grad=False)
416
+ # If rocm, normalize the weights and scales to e4m3fnuz
417
+ if is_hip():
418
+ # Normalize the weights and scales
419
+ w13_weight, w13_weight_scale, w13_input_scale = \
420
+ normalize_e4m3fn_to_e4m3fnuz(
421
+ layer.w13_weight, layer.w13_weight_scale,
422
+ layer.w13_input_scale)
423
+ w2_weight, w2_weight_scale, w2_input_scale = \
424
+ normalize_e4m3fn_to_e4m3fnuz(
425
+ layer.w2_weight, layer.w2_weight_scale,
426
+ layer.w2_input_scale)
427
+ # Reset the parameter
428
+ layer.w13_weight = torch.nn.Parameter(w13_weight,
429
+ requires_grad=False)
430
+ layer.w13_weight_scale = torch.nn.Parameter(
431
+ w13_weight_scale, requires_grad=False)
432
+ if w13_input_scale is not None:
433
+ layer.w13_input_scale = torch.nn.Parameter(
434
+ w13_input_scale, requires_grad=False)
435
+ layer.w2_weight = torch.nn.Parameter(w2_weight,
436
+ requires_grad=False)
437
+ layer.w2_weight_scale = torch.nn.Parameter(w2_weight_scale,
438
+ requires_grad=False)
439
+ if w2_input_scale is not None:
440
+ layer.w2_input_scale = torch.nn.Parameter(
441
+ w2_input_scale, requires_grad=False)
442
443
# Fp8 moe kernel needs single weight scale for w13 per expert.
444
# We take the max then dequant and requant each expert.
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+47
-7
@@ -6,9 +6,19 @@ from torch.nn import Parameter
6
from vllm import _custom_ops as ops
7
from vllm.model_executor.utils import set_weight_attrs
8
from vllm.platforms import current_platform
9
+from vllm.utils import is_hip
10
+
11
+# scaled_mm in pytorch on rocm has a bug that requires always
12
+# providing scaling factor for result. This value is created
13
+# as global value to avoid multiple tensor allocations, and
14
+# can be removed once pytorch fixes the bug.
15
+TORCH_SCALED_MM_SCALE_RESULT = torch.ones(1).cuda() if is_hip() else None
16
17
18
def cutlass_fp8_supported() -> bool:
19
+ # cutlass is not supported on Rocm
20
+ if is_hip():
21
+ return False
22
capability = current_platform.get_device_capability()
23
capability = capability[0] * 10 + capability[1]
24
@@ -147,13 +157,19 @@ def apply_fp8_linear(
157
158
if per_tensor_weights and per_tensor_activations:
159
# Fused GEMM_DQ
150
- output, _ = torch._scaled_mm(qinput,
151
- weight,
152
- out_dtype=input.dtype,
153
- scale_a=x_scale,
154
- scale_b=weight_scale,
155
- bias=bias)
156
- return torch.narrow(output, 0, 0, input.shape[0])
160
+ output = torch._scaled_mm(
161
+ qinput,
162
+ weight,
163
+ out_dtype=input.dtype,
164
+ scale_a=x_scale,
165
+ scale_b=weight_scale,
166
+ scale_result=TORCH_SCALED_MM_SCALE_RESULT,
167
+ bias=bias)
168
+ # Since in torch 2.5, scaled_mm only returns single value
169
+ # This should be removed when vllm-nvidia also moves to 2.5
170
+ if is_hip():
171
+ return torch.narrow(output, 0, 0, input.shape[0])
172
+ return torch.narrow(output[0], 0, 0, input.shape[0])
173
174
else:
175
# Fallback for channelwise case, where we use unfused DQ
@@ -207,3 +223,27 @@ def apply_int8_linear(
223
scale_b=weight_scale,
224
out_dtype=input.dtype,
225
bias=bias)
226
+
227
+
228
+def normalize_e4m3fn_to_e4m3fnuz(
229
+ weight: torch.Tensor,
230
+ weight_scale: torch.Tensor,
231
+ input_scale: Optional[torch.Tensor] = None
232
+) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
233
+ assert weight.dtype == torch.float8_e4m3fn
234
+ # The bits pattern 10000000(-128) represents zero in e4m3fn
235
+ # but NaN in e4m3fnuz. So here we set it to 0.
236
+ # https://onnx.ai/onnx/technical/float8.html
237
+ weight_as_int8 = weight.view(torch.int8)
238
+ ROCM_FP8_NAN_AS_INT = -128
239
+ weight_as_int8[weight_as_int8 == ROCM_FP8_NAN_AS_INT] = 0
240
+ weight = weight_as_int8.view(torch.float8_e4m3fnuz)
241
+
242
+ # For the same bits representation, e4m3fnuz value is half of
243
+ # the e4m3fn value, so we should double the scaling factor to
244
+ # get the same dequantized value.
245
+ # https://onnx.ai/onnx/technical/float8.html
246
+ weight_scale = weight_scale * 2.0
247
+ if input_scale is not None:
248
+ input_scale = input_scale * 2.0
249
+ return weight, weight_scale, input_scale