@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b5241e41d9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[ Kernel ] FP8 Dynamic-Per-Token Quant Kernel (#6511)

Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>

Varun Sundar Rabindranath committed Jul 17, 2024 at 21:38 UTC b5241e41d9fef56a89dcfda367a7eff87a07e3f7
7 files changed +271 -40
csrc/ops.h
+7 -3
@@ -128,12 +128,16 @@ torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
128
129 void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int64_t bit);
130
131 -void static_scaled_fp8_quant(torch::Tensor& out, torch::Tensor& input,
132 - torch::Tensor& scale);
131 +void static_scaled_fp8_quant(torch::Tensor& out, torch::Tensor const& input,
132 + torch::Tensor const& scale);
133
134 -void dynamic_scaled_fp8_quant(torch::Tensor& out, torch::Tensor& input,
134 +void dynamic_scaled_fp8_quant(torch::Tensor& out, torch::Tensor const& input,
135 torch::Tensor& scale);
136
137 +void dynamic_per_token_scaled_fp8_quant(torch::Tensor& out,
138 + torch::Tensor const& input,
139 + torch::Tensor& scale);
140 +
141 void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
142 int64_t block_size, torch::Tensor sorted_token_ids,
143 torch::Tensor experts_ids,
csrc/quantization/fp8/common.cu
+124 -20
@@ -7,6 +7,8 @@
7 #include "cuda_compat.h"
8 #include "dispatch_utils.h"
9
10 +#include "../../reduction_utils.cuh"
11 +
12 namespace vllm {
13
14 __device__ __forceinline__ float atomicMaxFloat(float* addr, float value) {
@@ -88,25 +90,48 @@ typedef struct __align__(4) {
90 float8x4_t;
91
92 template <typename scalar_t>
91 -__global__ void scaled_fp8_quant_kernel(c10::Float8_e4m3fn* __restrict__ out,
92 - const scalar_t* __restrict__ input,
93 - const float* __restrict__ scale,
94 - int64_t num_elems) {
95 - int tid = blockDim.x * blockIdx.x + threadIdx.x;
93 +__device__ float thread_max_vec(scalar_t const* __restrict__ input,
94 + int64_t const num_elems, int const tid,
95 + int const step) {
96 + // Vectorized input/output to better utilize memory bandwidth.
97 + vec4_t<scalar_t> const* vectorized_in =
98 + reinterpret_cast<vec4_t<scalar_t> const*>(input);
99
97 - // Invert the scale so that we can use multiplications to avoid expensive
98 - // division.
99 - const float inverted_scale = 1.0f / (*scale);
100 + int const num_vec_elems = num_elems >> 2;
101 + float absmax_val = 0.0f;
102 +
103 +#pragma unroll 4
104 + for (int i = tid; i < num_vec_elems; i += step) {
105 + vec4_t<scalar_t> in_vec = vectorized_in[i];
106 + absmax_val = max(absmax_val, fabs(in_vec.x));
107 + absmax_val = max(absmax_val, fabs(in_vec.y));
108 + absmax_val = max(absmax_val, fabs(in_vec.z));
109 + absmax_val = max(absmax_val, fabs(in_vec.w));
110 + }
111
112 + // Handle the remaining elements if num_elems is not divisible by 4
113 + for (int i = num_vec_elems * 4 + tid; i < num_elems; i += step) {
114 + absmax_val = max(absmax_val, fabs(input[i]));
115 + }
116 +
117 + return absmax_val;
118 +}
119 +
120 +template <typename scalar_t>
121 +__device__ void scaled_fp8_conversion_vec(c10::Float8_e4m3fn* __restrict__ out,
122 + scalar_t const* __restrict__ input,
123 + float const inverted_scale,
124 + int64_t const num_elems,
125 + int const tid, int const step) {
126 // Vectorized input/output to better utilize memory bandwidth.
102 - const vec4_t<scalar_t>* vectorized_in =
103 - reinterpret_cast<const vec4_t<scalar_t>*>(input);
127 + vec4_t<scalar_t> const* vectorized_in =
128 + reinterpret_cast<vec4_t<scalar_t> const*>(input);
129 float8x4_t* vectorized_out = reinterpret_cast<float8x4_t*>(out);
130
106 - int num_vec_elems = num_elems >> 2;
131 + int const num_vec_elems = num_elems >> 2;
132
133 #pragma unroll 4
109 - for (int i = tid; i < num_vec_elems; i += blockDim.x * gridDim.x) {
134 + for (int i = tid; i < num_vec_elems; i += step) {
135 vec4_t<scalar_t> in_vec = vectorized_in[i];
136 float8x4_t out_vec;
137
@@ -118,17 +143,74 @@ __global__ void scaled_fp8_quant_kernel(c10::Float8_e4m3fn* __restrict__ out,
143 }
144
145 // Handle the remaining elements if num_elems is not divisible by 4
121 - for (int i = num_vec_elems * 4 + tid; i < num_elems;
122 - i += blockDim.x * gridDim.x) {
146 + for (int i = num_vec_elems * 4 + tid; i < num_elems; i += step) {
147 out[i] = scaled_fp8_conversion(input[i], inverted_scale);
148 }
149 }
150
151 +template <typename scalar_t>
152 +__global__ void scaled_fp8_quant_kernel(c10::Float8_e4m3fn* __restrict__ out,
153 + const scalar_t* __restrict__ input,
154 + const float* __restrict__ scale,
155 + int64_t num_elems) {
156 + int tid = blockDim.x * blockIdx.x + threadIdx.x;
157 +
158 + // Invert the scale so that we can use multiplications to avoid expensive
159 + // division.
160 + const float inverted_scale = 1.0f / (*scale);
161 +
162 + scaled_fp8_conversion_vec(out, input, inverted_scale, num_elems, tid,
163 + blockDim.x * gridDim.x);
164 +}
165 +
166 +template <typename scalar_t>
167 +__global__ void dynamic_per_token_scaled_fp8_quant_kernel(
168 + c10::Float8_e4m3fn* __restrict__ out, float* __restrict__ scale,
169 + scalar_t const* __restrict__ input, const int hidden_size) {
170 + int const tid = threadIdx.x;
171 + int const token_idx = blockIdx.x;
172 +
173 + scalar_t const* __restrict__ token_input = &input[token_idx * hidden_size];
174 + c10::Float8_e4m3fn* __restrict__ token_output = &out[token_idx * hidden_size];
175 +
176 + // For vectorization, token_input and token_output pointers need to be
177 + // aligned at 8-byte and 4-byte addresses respectively.
178 + bool const can_vectorize = hidden_size % 4 == 0;
179 +
180 + float absmax_val = 0.0f;
181 + if (can_vectorize) {
182 + absmax_val = thread_max_vec(token_input, hidden_size, tid, blockDim.x);
183 + } else {
184 + for (int i = tid; i < hidden_size; i += blockDim.x) {
185 + float const x = static_cast<float>(token_input[i]);
186 + absmax_val = max(absmax_val, fabs(x));
187 + }
188 + }
189 +
190 + float const block_absmax_val_maybe = blockReduceMax(absmax_val);
191 + __shared__ float block_absmax_val;
192 + if (tid == 0) {
193 + block_absmax_val = block_absmax_val_maybe;
194 + scale[token_idx] = block_absmax_val / FP8_E4M3_MAX;
195 + }
196 + __syncthreads();
197 +
198 + float const inverted_scale = FP8_E4M3_MAX / block_absmax_val;
199 + if (can_vectorize) {
200 + scaled_fp8_conversion_vec(token_output, token_input, inverted_scale,
201 + hidden_size, tid, blockDim.x);
202 + } else {
203 + for (int i = tid; i < hidden_size; i += blockDim.x) {
204 + token_output[i] = scaled_fp8_conversion(token_input[i], inverted_scale);
205 + }
206 + }
207 +}
208 +
209 } // namespace vllm
210
129 -void static_scaled_fp8_quant(torch::Tensor& out, // [..., d]
130 - torch::Tensor& input, // [..., d]
131 - torch::Tensor& scale) // [1]
211 +void static_scaled_fp8_quant(torch::Tensor& out, // [..., d]
212 + torch::Tensor const& input, // [..., d]
213 + torch::Tensor const& scale) // [1]
214 {
215 int64_t num_tokens = input.numel() / input.size(-1);
216 int64_t num_elems = input.numel();
@@ -144,9 +226,9 @@ void static_scaled_fp8_quant(torch::Tensor& out, // [..., d]
226 });
227 }
228
147 -void dynamic_scaled_fp8_quant(torch::Tensor& out, // [..., d]
148 - torch::Tensor& input, // [..., d]
149 - torch::Tensor& scale) // [1]
229 +void dynamic_scaled_fp8_quant(torch::Tensor& out, // [..., d]
230 + torch::Tensor const& input, // [..., d]
231 + torch::Tensor& scale) // [1]
232 {
233 int64_t num_tokens = input.numel() / input.size(-1);
234 int64_t num_elems = input.numel();
@@ -163,3 +245,25 @@ void dynamic_scaled_fp8_quant(torch::Tensor& out, // [..., d]
245 scale.data_ptr<float>(), num_elems);
246 });
247 }
248 +
249 +void dynamic_per_token_scaled_fp8_quant(torch::Tensor& out, // [..., d]
250 + torch::Tensor const& input, // [..., d]
251 + torch::Tensor& scales) {
252 + TORCH_CHECK(input.is_contiguous());
253 + TORCH_CHECK(out.is_contiguous());
254 +
255 + int const hidden_size = input.size(-1);
256 + int const num_tokens = input.numel() / hidden_size;
257 + dim3 const grid(num_tokens);
258 + dim3 const block(std::min(hidden_size, 1024));
259 +
260 + const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
261 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
262 + VLLM_DISPATCH_FLOATING_TYPES(
263 + input.scalar_type(), "dynamic_per_token_scaled_fp8_quant_kernel", [&] {
264 + vllm::dynamic_per_token_scaled_fp8_quant_kernel<scalar_t>
265 + <<<grid, block, 0, stream>>>(
266 + out.data_ptr<c10::Float8_e4m3fn>(), scales.data_ptr<float>(),
267 + input.data_ptr<scalar_t>(), hidden_size);
268 + });
269 +}
csrc/torch_bindings.cpp
+9 -1
@@ -179,12 +179,20 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
179 "static_scaled_fp8_quant(Tensor! out, Tensor input, Tensor scale) -> ()");
180 ops.impl("static_scaled_fp8_quant", torch::kCUDA, &static_scaled_fp8_quant);
181
182 - // Compute FP8 quantized tensor and scaling factor.
182 + // Compute dynamic-per-tensor FP8 quantized tensor and scaling factor.
183 ops.def(
184 "dynamic_scaled_fp8_quant(Tensor! out, Tensor input, Tensor! scale) -> "
185 "()");
186 ops.impl("dynamic_scaled_fp8_quant", torch::kCUDA, &dynamic_scaled_fp8_quant);
187
188 + // Compute dynamic-per-token FP8 quantized tensor and scaling factor.
189 + ops.def(
190 + "dynamic_per_token_scaled_fp8_quant(Tensor! out, Tensor input, Tensor! "
191 + "scale) -> "
192 + "()");
193 + ops.impl("dynamic_per_token_scaled_fp8_quant", torch::kCUDA,
194 + &dynamic_per_token_scaled_fp8_quant);
195 +
196 // Aligning the number of tokens to be processed by each expert such
197 // that it is divisible by the block size.
198 ops.def(
tests/kernels/quant_utils.py new
+56
@@ -0,0 +1,56 @@
1 +from typing import Tuple, Union
2 +
3 +import torch
4 +
5 +
6 +def as_float32_tensor(x: Union[float, torch.tensor]) -> torch.tensor:
7 + return torch.as_tensor(x, dtype=torch.float32, device='cuda')
8 +
9 +def ref_dynamic_per_token_quant(x: torch.tensor,
10 + quant_dtype: torch.dtype) \
11 + -> Tuple[torch.tensor, torch.tensor]:
12 +
13 + assert quant_dtype in [torch.int8, torch.float8_e4m3fn]
14 + qtype_traits = torch.iinfo(quant_dtype) if quant_dtype == torch.int8 \
15 + else torch.finfo(quant_dtype)
16 + qtype_max = as_float32_tensor(qtype_traits.max)
17 +
18 + # For fp8, in order to match the cuda kernel output, we have to do exactly
19 + # the same operations as in the corresponding fp8 kernel to prevent
20 + # rounding errors.
21 +
22 + # Compute scales
23 + x_token_max, _ = x.abs().max(dim=-1)
24 + x_token_max = as_float32_tensor(x_token_max)
25 + scales = (x_token_max / qtype_max)[:, None]
26 +
27 + # Quant
28 + iscales = (qtype_max / x_token_max)[:, None]
29 + torch_out = as_float32_tensor(x) * iscales
30 + torch_out = torch_out.round() if quant_dtype == torch.int8 else torch_out
31 + torch_out = torch_out.clamp(qtype_traits.min,
32 + qtype_traits.max).to(quant_dtype)
33 +
34 + return torch_out, scales
35 +
36 +
37 +# The int8 version is very similar. Incorporate the int8 version, like in
38 +# ref_dynamic_per_token_quant, when we have a dynamic_per_tensor int8 quant
39 +# kernel
40 +def ref_dynamic_per_tensor_fp8_quant(x: torch.tensor) \
41 + -> Tuple[torch.tensor, torch.tensor]:
42 +
43 + fp8_traits = torch.finfo(torch.float8_e4m3fn)
44 + fp8_max = as_float32_tensor(fp8_traits.max)
45 + one = as_float32_tensor(1.0)
46 +
47 + # For fp8, in order to match the cuda kernel output, we have to do exactly
48 + # the same operations as in the corresponding fp8 kernel to prevent
49 + # rounding errors.
50 +
51 + x_max = as_float32_tensor(x.abs().max())
52 + ref_scale = x_max / fp8_max
53 + ref_iscale = one / ref_scale
54 + ref_out = (as_float32_tensor(x) * ref_iscale).clamp(
55 + fp8_traits.min, fp8_traits.max).to(dtype=torch.float8_e4m3fn)
56 + return ref_out, ref_scale
tests/kernels/test_fp8_quant.py new
+54
@@ -0,0 +1,54 @@
1 +import pytest
2 +import torch
3 +
4 +import vllm._custom_ops as ops
5 +from tests.kernels.quant_utils import (ref_dynamic_per_tensor_fp8_quant,
6 + ref_dynamic_per_token_quant)
7 +
8 +DTYPES = [torch.half, torch.bfloat16, torch.float]
9 +HIDDEN_SIZES = [1, 2, 3, 4, 16, 67, 768, 2048, 5120, 5137, 8192,
10 + 8193] # Arbitrary values for testing
11 +HIDDEN_SIZES += list(range(1024, 1033)) # vectorized conversion edge cases
12 +NUM_TOKENS = [1, 7, 83, 4096] # Arbitrary values for testing
13 +SEEDS = [0]
14 +
15 +
16 +@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
17 +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
18 +@pytest.mark.parametrize("dtype", DTYPES)
19 +@pytest.mark.parametrize("seed", SEEDS)
20 +@torch.inference_mode()
21 +def test_dynamic_per_token_fp8_quant(num_tokens: int, hidden_size: int,
22 + dtype: torch.dtype, seed: int) -> None:
23 + torch.random.manual_seed(seed)
24 + torch.cuda.manual_seed(seed)
25 +
26 + x = torch.rand(num_tokens, hidden_size, dtype=dtype,
27 + device="cuda") + 1e-6 # avoid nans
28 +
29 + ref_out, ref_scales = ref_dynamic_per_token_quant(x, torch.float8_e4m3fn)
30 + ops_out, ops_scales = ops.dynamic_per_token_scaled_fp8_quant(x)
31 +
32 + assert torch.allclose(ref_scales, ops_scales)
33 + assert torch.allclose(ref_out.to(dtype=torch.float32),
34 + ops_out.to(dtype=torch.float32))
35 +
36 +
37 +@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
38 +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
39 +@pytest.mark.parametrize("dtype", DTYPES)
40 +@pytest.mark.parametrize("seed", SEEDS)
41 +@torch.inference_mode()
42 +def test_dynamic_per_tensor_fp8_quant(num_tokens: int, hidden_size: int,
43 + dtype: torch.dtype, seed: int) -> None:
44 + torch.random.manual_seed(seed)
45 + torch.cuda.manual_seed(seed)
46 +
47 + x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
48 +
49 + ref_out, ref_scale = ref_dynamic_per_tensor_fp8_quant(x)
50 + ops_out, ops_scale = ops.scaled_fp8_quant(x)
51 +
52 + assert torch.allclose(ref_scale, ops_scale)
53 + assert torch.allclose(ref_out.to(dtype=torch.float32),
54 + ops_out.to(dtype=torch.float32))
tests/kernels/test_int8_quant.py
+10 -16
@@ -3,6 +3,8 @@ import torch
3
4 # ruff: noqa: F401
5 import vllm._C
6 +from tests.kernels.quant_utils import ref_dynamic_per_token_quant
7 +from vllm._custom_ops import scaled_int8_quant
8
9 DTYPES = [torch.half, torch.bfloat16, torch.float]
10 HIDDEN_SIZES = [16, 67, 768, 2048, 5120, 5137, 8192,
@@ -21,23 +23,16 @@ def test_dynamic_scaled_int8_quant(num_tokens: int, hidden_size: int,
23 dtype: torch.dtype, seed: int) -> None:
24 torch.random.manual_seed(seed)
25 torch.cuda.manual_seed(seed)
24 - int8_traits = torch.iinfo(torch.int8)
26
27 x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") * 1000
28
28 - x_token_max, _ = x.max(dim=1)
29 - x_token_max = x_token_max.to(dtype=torch.float32)
30 - scales = (x_token_max / float(127.0))[:, None].to(device="cuda",
31 - dtype=torch.float32)
32 - torch_out = (x / scales).round().clamp(int8_traits.min,
33 - int8_traits.max).to(torch.int8)
34 -
35 - ops_out = torch.empty_like(x, dtype=torch.int8, device="cuda")
36 - scales_out = torch.empty_like(scales, dtype=torch.float32, device="cuda")
37 - torch.ops._C.dynamic_scaled_int8_quant(ops_out, x, scales_out)
29 + # reference
30 + ref_out, ref_scales = ref_dynamic_per_token_quant(x, torch.int8)
31 + # kernel
32 + ops_out, ops_scales = scaled_int8_quant(x)
33
39 - assert torch.allclose(scales_out, scales)
40 - assert torch.allclose(torch_out, ops_out,
34 + assert torch.allclose(ops_scales, ref_scales)
35 + assert torch.allclose(ops_out, ref_out,
36 atol=1) # big atol to account for rounding errors
37
38
@@ -55,12 +50,11 @@ def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
50 int8_traits = torch.iinfo(torch.int8)
51
52 x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") * 1000
53 + scale = torch.tensor([scale], dtype=torch.float32, device="cuda")
54
55 out1 = (x / scale).round().clamp(int8_traits.min,
56 int8_traits.max).to(torch.int8)
61 - out2 = torch.empty_like(x, dtype=torch.int8)
62 - scale_argument = torch.tensor([scale], dtype=torch.float32, device="cuda")
57 + out2, _ = scaled_int8_quant(x, scale)
58
64 - torch.ops._C.static_scaled_int8_quant(out2, x, scale_argument)
59 assert torch.allclose(out1, out2,
60 atol=1) # big atol to account for rounding errors
vllm/_custom_ops.py
+11
@@ -335,6 +335,17 @@ def scaled_fp8_quant(
335 return output, scale
336
337
338 +def dynamic_per_token_scaled_fp8_quant(
339 + input: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
340 +
341 + output = torch.empty_like(input, dtype=torch.float8_e4m3fn)
342 + scales = torch.empty((input.numel() // input.shape[-1], 1),
343 + device=input.device,
344 + dtype=torch.float32)
345 + torch.ops._C.dynamic_per_token_scaled_fp8_quant(output, input, scales)
346 + return output, scales
347 +
348 +
349 # int8
350 def scaled_int8_quant(
351 input: torch.Tensor,