A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[ Kernel ] FP8 Dynamic-Per-Token Quant Kernel (#6511)
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Jul 17, 2024 at 21:38 UTC
b5241e41d9fef56a89dcfda367a7eff87a07e3f7
7 files changed
+271
-40
csrc/ops.h
+7
-3
@@ -128,12 +128,16 @@ torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
128
129
void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int64_t bit);
130
131
-void static_scaled_fp8_quant(torch::Tensor& out, torch::Tensor& input,
132
- torch::Tensor& scale);
131
+void static_scaled_fp8_quant(torch::Tensor& out, torch::Tensor const& input,
132
+ torch::Tensor const& scale);
133
134
-void dynamic_scaled_fp8_quant(torch::Tensor& out, torch::Tensor& input,
134
+void dynamic_scaled_fp8_quant(torch::Tensor& out, torch::Tensor const& input,
135
torch::Tensor& scale);
136
137
+void dynamic_per_token_scaled_fp8_quant(torch::Tensor& out,
138
+ torch::Tensor const& input,
139
+ torch::Tensor& scale);
140
+
141
void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
142
int64_t block_size, torch::Tensor sorted_token_ids,
143
torch::Tensor experts_ids,
csrc/quantization/fp8/common.cu
+124
-20
@@ -7,6 +7,8 @@
7
#include "cuda_compat.h"
8
#include "dispatch_utils.h"
9
10
+#include "../../reduction_utils.cuh"
11
+
12
namespace vllm {
13
14
__device__ __forceinline__ float atomicMaxFloat(float* addr, float value) {
@@ -88,25 +90,48 @@ typedef struct __align__(4) {
90
float8x4_t;
91
92
template <typename scalar_t>
91
-__global__ void scaled_fp8_quant_kernel(c10::Float8_e4m3fn* __restrict__ out,
92
- const scalar_t* __restrict__ input,
93
- const float* __restrict__ scale,
94
- int64_t num_elems) {
95
- int tid = blockDim.x * blockIdx.x + threadIdx.x;
93
+__device__ float thread_max_vec(scalar_t const* __restrict__ input,
94
+ int64_t const num_elems, int const tid,
95
+ int const step) {
96
+ // Vectorized input/output to better utilize memory bandwidth.
97
+ vec4_t<scalar_t> const* vectorized_in =
98
+ reinterpret_cast<vec4_t<scalar_t> const*>(input);
99
97
- // Invert the scale so that we can use multiplications to avoid expensive
98
- // division.
99
- const float inverted_scale = 1.0f / (*scale);
100
+ int const num_vec_elems = num_elems >> 2;
101
+ float absmax_val = 0.0f;
102
+
103
+#pragma unroll 4
104
+ for (int i = tid; i < num_vec_elems; i += step) {
105
+ vec4_t<scalar_t> in_vec = vectorized_in[i];
106
+ absmax_val = max(absmax_val, fabs(in_vec.x));
107
+ absmax_val = max(absmax_val, fabs(in_vec.y));
108
+ absmax_val = max(absmax_val, fabs(in_vec.z));
109
+ absmax_val = max(absmax_val, fabs(in_vec.w));
110
+ }
111
112
+ // Handle the remaining elements if num_elems is not divisible by 4
113
+ for (int i = num_vec_elems * 4 + tid; i < num_elems; i += step) {
114
+ absmax_val = max(absmax_val, fabs(input[i]));
115
+ }
116
+
117
+ return absmax_val;
118
+}
119
+
120
+template <typename scalar_t>
121
+__device__ void scaled_fp8_conversion_vec(c10::Float8_e4m3fn* __restrict__ out,
122
+ scalar_t const* __restrict__ input,
123
+ float const inverted_scale,
124
+ int64_t const num_elems,
125
+ int const tid, int const step) {
126
// Vectorized input/output to better utilize memory bandwidth.
102
- const vec4_t<scalar_t>* vectorized_in =
103
- reinterpret_cast<const vec4_t<scalar_t>*>(input);
127
+ vec4_t<scalar_t> const* vectorized_in =
128
+ reinterpret_cast<vec4_t<scalar_t> const*>(input);
129
float8x4_t* vectorized_out = reinterpret_cast<float8x4_t*>(out);
130
106
- int num_vec_elems = num_elems >> 2;
131
+ int const num_vec_elems = num_elems >> 2;
132
133
#pragma unroll 4
109
- for (int i = tid; i < num_vec_elems; i += blockDim.x * gridDim.x) {
134
+ for (int i = tid; i < num_vec_elems; i += step) {
135
vec4_t<scalar_t> in_vec = vectorized_in[i];
136
float8x4_t out_vec;
137
@@ -118,17 +143,74 @@ __global__ void scaled_fp8_quant_kernel(c10::Float8_e4m3fn* __restrict__ out,
143
}
144
145
// Handle the remaining elements if num_elems is not divisible by 4
121
- for (int i = num_vec_elems * 4 + tid; i < num_elems;
122
- i += blockDim.x * gridDim.x) {
146
+ for (int i = num_vec_elems * 4 + tid; i < num_elems; i += step) {
147
out[i] = scaled_fp8_conversion(input[i], inverted_scale);
148
}
149
}
150
151
+template <typename scalar_t>
152
+__global__ void scaled_fp8_quant_kernel(c10::Float8_e4m3fn* __restrict__ out,
153
+ const scalar_t* __restrict__ input,
154
+ const float* __restrict__ scale,
155
+ int64_t num_elems) {
156
+ int tid = blockDim.x * blockIdx.x + threadIdx.x;
157
+
158
+ // Invert the scale so that we can use multiplications to avoid expensive
159
+ // division.
160
+ const float inverted_scale = 1.0f / (*scale);
161
+
162
+ scaled_fp8_conversion_vec(out, input, inverted_scale, num_elems, tid,
163
+ blockDim.x * gridDim.x);
164
+}
165
+
166
+template <typename scalar_t>
167
+__global__ void dynamic_per_token_scaled_fp8_quant_kernel(
168
+ c10::Float8_e4m3fn* __restrict__ out, float* __restrict__ scale,
169
+ scalar_t const* __restrict__ input, const int hidden_size) {
170
+ int const tid = threadIdx.x;
171
+ int const token_idx = blockIdx.x;
172
+
173
+ scalar_t const* __restrict__ token_input = &input[token_idx * hidden_size];
174
+ c10::Float8_e4m3fn* __restrict__ token_output = &out[token_idx * hidden_size];
175
+
176
+ // For vectorization, token_input and token_output pointers need to be
177
+ // aligned at 8-byte and 4-byte addresses respectively.
178
+ bool const can_vectorize = hidden_size % 4 == 0;
179
+
180
+ float absmax_val = 0.0f;
181
+ if (can_vectorize) {
182
+ absmax_val = thread_max_vec(token_input, hidden_size, tid, blockDim.x);
183
+ } else {
184
+ for (int i = tid; i < hidden_size; i += blockDim.x) {
185
+ float const x = static_cast<float>(token_input[i]);
186
+ absmax_val = max(absmax_val, fabs(x));
187
+ }
188
+ }
189
+
190
+ float const block_absmax_val_maybe = blockReduceMax(absmax_val);
191
+ __shared__ float block_absmax_val;
192
+ if (tid == 0) {
193
+ block_absmax_val = block_absmax_val_maybe;
194
+ scale[token_idx] = block_absmax_val / FP8_E4M3_MAX;
195
+ }
196
+ __syncthreads();
197
+
198
+ float const inverted_scale = FP8_E4M3_MAX / block_absmax_val;
199
+ if (can_vectorize) {
200
+ scaled_fp8_conversion_vec(token_output, token_input, inverted_scale,
201
+ hidden_size, tid, blockDim.x);
202
+ } else {
203
+ for (int i = tid; i < hidden_size; i += blockDim.x) {
204
+ token_output[i] = scaled_fp8_conversion(token_input[i], inverted_scale);
205
+ }
206
+ }
207
+}
208
+
209
} // namespace vllm
210
129
-void static_scaled_fp8_quant(torch::Tensor& out, // [..., d]
130
- torch::Tensor& input, // [..., d]
131
- torch::Tensor& scale) // [1]
211
+void static_scaled_fp8_quant(torch::Tensor& out, // [..., d]
212
+ torch::Tensor const& input, // [..., d]
213
+ torch::Tensor const& scale) // [1]
214
{
215
int64_t num_tokens = input.numel() / input.size(-1);
216
int64_t num_elems = input.numel();
@@ -144,9 +226,9 @@ void static_scaled_fp8_quant(torch::Tensor& out, // [..., d]
226
});
227
}
228
147
-void dynamic_scaled_fp8_quant(torch::Tensor& out, // [..., d]
148
- torch::Tensor& input, // [..., d]
149
- torch::Tensor& scale) // [1]
229
+void dynamic_scaled_fp8_quant(torch::Tensor& out, // [..., d]
230
+ torch::Tensor const& input, // [..., d]
231
+ torch::Tensor& scale) // [1]
232
{
233
int64_t num_tokens = input.numel() / input.size(-1);
234
int64_t num_elems = input.numel();
@@ -163,3 +245,25 @@ void dynamic_scaled_fp8_quant(torch::Tensor& out, // [..., d]
245
scale.data_ptr<float>(), num_elems);
246
});
247
}
248
+
249
+void dynamic_per_token_scaled_fp8_quant(torch::Tensor& out, // [..., d]
250
+ torch::Tensor const& input, // [..., d]
251
+ torch::Tensor& scales) {
252
+ TORCH_CHECK(input.is_contiguous());
253
+ TORCH_CHECK(out.is_contiguous());
254
+
255
+ int const hidden_size = input.size(-1);
256
+ int const num_tokens = input.numel() / hidden_size;
257
+ dim3 const grid(num_tokens);
258
+ dim3 const block(std::min(hidden_size, 1024));
259
+
260
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
261
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
262
+ VLLM_DISPATCH_FLOATING_TYPES(
263
+ input.scalar_type(), "dynamic_per_token_scaled_fp8_quant_kernel", [&] {
264
+ vllm::dynamic_per_token_scaled_fp8_quant_kernel<scalar_t>
265
+ <<<grid, block, 0, stream>>>(
266
+ out.data_ptr<c10::Float8_e4m3fn>(), scales.data_ptr<float>(),
267
+ input.data_ptr<scalar_t>(), hidden_size);
268
+ });
269
+}
csrc/torch_bindings.cpp
+9
-1
@@ -179,12 +179,20 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
179
"static_scaled_fp8_quant(Tensor! out, Tensor input, Tensor scale) -> ()");
180
ops.impl("static_scaled_fp8_quant", torch::kCUDA, &static_scaled_fp8_quant);
181
182
- // Compute FP8 quantized tensor and scaling factor.
182
+ // Compute dynamic-per-tensor FP8 quantized tensor and scaling factor.
183
ops.def(
184
"dynamic_scaled_fp8_quant(Tensor! out, Tensor input, Tensor! scale) -> "
185
"()");
186
ops.impl("dynamic_scaled_fp8_quant", torch::kCUDA, &dynamic_scaled_fp8_quant);
187
188
+ // Compute dynamic-per-token FP8 quantized tensor and scaling factor.
189
+ ops.def(
190
+ "dynamic_per_token_scaled_fp8_quant(Tensor! out, Tensor input, Tensor! "
191
+ "scale) -> "
192
+ "()");
193
+ ops.impl("dynamic_per_token_scaled_fp8_quant", torch::kCUDA,
194
+ &dynamic_per_token_scaled_fp8_quant);
195
+
196
// Aligning the number of tokens to be processed by each expert such
197
// that it is divisible by the block size.
198
ops.def(
tests/kernels/quant_utils.py
new
+56
@@ -0,0 +1,56 @@
1
+from typing import Tuple, Union
2
+
3
+import torch
4
+
5
+
6
+def as_float32_tensor(x: Union[float, torch.tensor]) -> torch.tensor:
7
+ return torch.as_tensor(x, dtype=torch.float32, device='cuda')
8
+
9
+def ref_dynamic_per_token_quant(x: torch.tensor,
10
+ quant_dtype: torch.dtype) \
11
+ -> Tuple[torch.tensor, torch.tensor]:
12
+
13
+ assert quant_dtype in [torch.int8, torch.float8_e4m3fn]
14
+ qtype_traits = torch.iinfo(quant_dtype) if quant_dtype == torch.int8 \
15
+ else torch.finfo(quant_dtype)
16
+ qtype_max = as_float32_tensor(qtype_traits.max)
17
+
18
+ # For fp8, in order to match the cuda kernel output, we have to do exactly
19
+ # the same operations as in the corresponding fp8 kernel to prevent
20
+ # rounding errors.
21
+
22
+ # Compute scales
23
+ x_token_max, _ = x.abs().max(dim=-1)
24
+ x_token_max = as_float32_tensor(x_token_max)
25
+ scales = (x_token_max / qtype_max)[:, None]
26
+
27
+ # Quant
28
+ iscales = (qtype_max / x_token_max)[:, None]
29
+ torch_out = as_float32_tensor(x) * iscales
30
+ torch_out = torch_out.round() if quant_dtype == torch.int8 else torch_out
31
+ torch_out = torch_out.clamp(qtype_traits.min,
32
+ qtype_traits.max).to(quant_dtype)
33
+
34
+ return torch_out, scales
35
+
36
+
37
+# The int8 version is very similar. Incorporate the int8 version, like in
38
+# ref_dynamic_per_token_quant, when we have a dynamic_per_tensor int8 quant
39
+# kernel
40
+def ref_dynamic_per_tensor_fp8_quant(x: torch.tensor) \
41
+ -> Tuple[torch.tensor, torch.tensor]:
42
+
43
+ fp8_traits = torch.finfo(torch.float8_e4m3fn)
44
+ fp8_max = as_float32_tensor(fp8_traits.max)
45
+ one = as_float32_tensor(1.0)
46
+
47
+ # For fp8, in order to match the cuda kernel output, we have to do exactly
48
+ # the same operations as in the corresponding fp8 kernel to prevent
49
+ # rounding errors.
50
+
51
+ x_max = as_float32_tensor(x.abs().max())
52
+ ref_scale = x_max / fp8_max
53
+ ref_iscale = one / ref_scale
54
+ ref_out = (as_float32_tensor(x) * ref_iscale).clamp(
55
+ fp8_traits.min, fp8_traits.max).to(dtype=torch.float8_e4m3fn)
56
+ return ref_out, ref_scale
tests/kernels/test_fp8_quant.py
new
+54
@@ -0,0 +1,54 @@
1
+import pytest
2
+import torch
3
+
4
+import vllm._custom_ops as ops
5
+from tests.kernels.quant_utils import (ref_dynamic_per_tensor_fp8_quant,
6
+ ref_dynamic_per_token_quant)
7
+
8
+DTYPES = [torch.half, torch.bfloat16, torch.float]
9
+HIDDEN_SIZES = [1, 2, 3, 4, 16, 67, 768, 2048, 5120, 5137, 8192,
10
+ 8193] # Arbitrary values for testing
11
+HIDDEN_SIZES += list(range(1024, 1033)) # vectorized conversion edge cases
12
+NUM_TOKENS = [1, 7, 83, 4096] # Arbitrary values for testing
13
+SEEDS = [0]
14
+
15
+
16
+@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
17
+@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
18
+@pytest.mark.parametrize("dtype", DTYPES)
19
+@pytest.mark.parametrize("seed", SEEDS)
20
+@torch.inference_mode()
21
+def test_dynamic_per_token_fp8_quant(num_tokens: int, hidden_size: int,
22
+ dtype: torch.dtype, seed: int) -> None:
23
+ torch.random.manual_seed(seed)
24
+ torch.cuda.manual_seed(seed)
25
+
26
+ x = torch.rand(num_tokens, hidden_size, dtype=dtype,
27
+ device="cuda") + 1e-6 # avoid nans
28
+
29
+ ref_out, ref_scales = ref_dynamic_per_token_quant(x, torch.float8_e4m3fn)
30
+ ops_out, ops_scales = ops.dynamic_per_token_scaled_fp8_quant(x)
31
+
32
+ assert torch.allclose(ref_scales, ops_scales)
33
+ assert torch.allclose(ref_out.to(dtype=torch.float32),
34
+ ops_out.to(dtype=torch.float32))
35
+
36
+
37
+@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
38
+@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
39
+@pytest.mark.parametrize("dtype", DTYPES)
40
+@pytest.mark.parametrize("seed", SEEDS)
41
+@torch.inference_mode()
42
+def test_dynamic_per_tensor_fp8_quant(num_tokens: int, hidden_size: int,
43
+ dtype: torch.dtype, seed: int) -> None:
44
+ torch.random.manual_seed(seed)
45
+ torch.cuda.manual_seed(seed)
46
+
47
+ x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
48
+
49
+ ref_out, ref_scale = ref_dynamic_per_tensor_fp8_quant(x)
50
+ ops_out, ops_scale = ops.scaled_fp8_quant(x)
51
+
52
+ assert torch.allclose(ref_scale, ops_scale)
53
+ assert torch.allclose(ref_out.to(dtype=torch.float32),
54
+ ops_out.to(dtype=torch.float32))
tests/kernels/test_int8_quant.py
+10
-16
@@ -3,6 +3,8 @@ import torch
3
4
# ruff: noqa: F401
5
import vllm._C
6
+from tests.kernels.quant_utils import ref_dynamic_per_token_quant
7
+from vllm._custom_ops import scaled_int8_quant
8
9
DTYPES = [torch.half, torch.bfloat16, torch.float]
10
HIDDEN_SIZES = [16, 67, 768, 2048, 5120, 5137, 8192,
@@ -21,23 +23,16 @@ def test_dynamic_scaled_int8_quant(num_tokens: int, hidden_size: int,
23
dtype: torch.dtype, seed: int) -> None:
24
torch.random.manual_seed(seed)
25
torch.cuda.manual_seed(seed)
24
- int8_traits = torch.iinfo(torch.int8)
26
27
x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") * 1000
28
28
- x_token_max, _ = x.max(dim=1)
29
- x_token_max = x_token_max.to(dtype=torch.float32)
30
- scales = (x_token_max / float(127.0))[:, None].to(device="cuda",
31
- dtype=torch.float32)
32
- torch_out = (x / scales).round().clamp(int8_traits.min,
33
- int8_traits.max).to(torch.int8)
34
-
35
- ops_out = torch.empty_like(x, dtype=torch.int8, device="cuda")
36
- scales_out = torch.empty_like(scales, dtype=torch.float32, device="cuda")
37
- torch.ops._C.dynamic_scaled_int8_quant(ops_out, x, scales_out)
29
+ # reference
30
+ ref_out, ref_scales = ref_dynamic_per_token_quant(x, torch.int8)
31
+ # kernel
32
+ ops_out, ops_scales = scaled_int8_quant(x)
33
39
- assert torch.allclose(scales_out, scales)
40
- assert torch.allclose(torch_out, ops_out,
34
+ assert torch.allclose(ops_scales, ref_scales)
35
+ assert torch.allclose(ops_out, ref_out,
36
atol=1) # big atol to account for rounding errors
37
38
@@ -55,12 +50,11 @@ def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
50
int8_traits = torch.iinfo(torch.int8)
51
52
x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") * 1000
53
+ scale = torch.tensor([scale], dtype=torch.float32, device="cuda")
54
55
out1 = (x / scale).round().clamp(int8_traits.min,
56
int8_traits.max).to(torch.int8)
61
- out2 = torch.empty_like(x, dtype=torch.int8)
62
- scale_argument = torch.tensor([scale], dtype=torch.float32, device="cuda")
57
+ out2, _ = scaled_int8_quant(x, scale)
58
64
- torch.ops._C.static_scaled_int8_quant(out2, x, scale_argument)
59
assert torch.allclose(out1, out2,
60
atol=1) # big atol to account for rounding errors
vllm/_custom_ops.py
+11
@@ -335,6 +335,17 @@ def scaled_fp8_quant(
335
return output, scale
336
337
338
+def dynamic_per_token_scaled_fp8_quant(
339
+ input: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
340
+
341
+ output = torch.empty_like(input, dtype=torch.float8_e4m3fn)
342
+ scales = torch.empty((input.numel() // input.shape[-1], 1),
343
+ device=input.device,
344
+ dtype=torch.float32)
345
+ torch.ops._C.dynamic_per_token_scaled_fp8_quant(output, input, scales)
346
+ return output, scales
347
+
348
+
349
# int8
350
def scaled_int8_quant(
351
input: torch.Tensor,