@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2e26564259
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[ Kernel ] FP8 Dynamic Per Token Quant - Add scale_ub (#6593)

Co-authored-by: Varun Sundar Rabindranth <varun@neuralmagic.com>

Varun Sundar Rabindranath committed Jul 19, 2024 at 21:15 UTC 2e26564259801dc6359c49bb044104d8d5373b57
6 files changed +86 -39
csrc/ops.h
+3 -3
@@ -134,9 +134,9 @@ void static_scaled_fp8_quant(torch::Tensor& out, torch::Tensor const& input,
134 void dynamic_scaled_fp8_quant(torch::Tensor& out, torch::Tensor const& input,
135 torch::Tensor& scale);
136
137 -void dynamic_per_token_scaled_fp8_quant(torch::Tensor& out,
138 - torch::Tensor const& input,
139 - torch::Tensor& scale);
137 +void dynamic_per_token_scaled_fp8_quant(
138 + torch::Tensor& out, torch::Tensor const& input, torch::Tensor& scale,
139 + c10::optional<torch::Tensor> const& scale_ub);
140
141 void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
142 int64_t block_size, torch::Tensor sorted_token_ids,
csrc/quantization/fp8/common.cu
+48 -25
@@ -23,10 +23,16 @@ __device__ __forceinline__ float atomicMaxFloat(float* addr, float value) {
23
24 #define FP8_E4M3_MAX std::numeric_limits<c10::Float8_e4m3fn>::max()
25
26 -template <typename scalar_t>
26 +template <bool is_scale_inverted>
27 __device__ __forceinline__ c10::Float8_e4m3fn scaled_fp8_conversion(
28 - const scalar_t val, const float inverted_scale) {
29 - float x = static_cast<float>(val) * inverted_scale;
28 + float const val, float const scale) {
29 + float x = 0.0f;
30 + if constexpr (is_scale_inverted) {
31 + x = val * scale;
32 + } else {
33 + x = val / scale;
34 + }
35 +
36 float r = fmax(-FP8_E4M3_MAX, fmin(x, FP8_E4M3_MAX));
37 return static_cast<c10::Float8_e4m3fn>(r);
38 }
@@ -117,10 +123,10 @@ __device__ float thread_max_vec(scalar_t const* __restrict__ input,
123 return absmax_val;
124 }
125
120 -template <typename scalar_t>
126 +template <typename scalar_t, bool is_scale_inverted>
127 __device__ void scaled_fp8_conversion_vec(c10::Float8_e4m3fn* __restrict__ out,
128 scalar_t const* __restrict__ input,
123 - float const inverted_scale,
129 + float const scale,
130 int64_t const num_elems,
131 int const tid, int const step) {
132 // Vectorized input/output to better utilize memory bandwidth.
@@ -135,16 +141,21 @@ __device__ void scaled_fp8_conversion_vec(c10::Float8_e4m3fn* __restrict__ out,
141 vec4_t<scalar_t> in_vec = vectorized_in[i];
142 float8x4_t out_vec;
143
138 - out_vec.x = scaled_fp8_conversion(in_vec.x, inverted_scale);
139 - out_vec.y = scaled_fp8_conversion(in_vec.y, inverted_scale);
140 - out_vec.z = scaled_fp8_conversion(in_vec.z, inverted_scale);
141 - out_vec.w = scaled_fp8_conversion(in_vec.w, inverted_scale);
144 + out_vec.x = scaled_fp8_conversion<is_scale_inverted>(
145 + static_cast<float>(in_vec.x), scale);
146 + out_vec.y = scaled_fp8_conversion<is_scale_inverted>(
147 + static_cast<float>(in_vec.y), scale);
148 + out_vec.z = scaled_fp8_conversion<is_scale_inverted>(
149 + static_cast<float>(in_vec.z), scale);
150 + out_vec.w = scaled_fp8_conversion<is_scale_inverted>(
151 + static_cast<float>(in_vec.w), scale);
152 vectorized_out[i] = out_vec;
153 }
154
155 // Handle the remaining elements if num_elems is not divisible by 4
156 for (int i = num_vec_elems * 4 + tid; i < num_elems; i += step) {
147 - out[i] = scaled_fp8_conversion(input[i], inverted_scale);
157 + out[i] = scaled_fp8_conversion<is_scale_inverted>(
158 + static_cast<float>(input[i]), scale);
159 }
160 }
161
@@ -158,15 +169,17 @@ __global__ void scaled_fp8_quant_kernel(c10::Float8_e4m3fn* __restrict__ out,
169 // Invert the scale so that we can use multiplications to avoid expensive
170 // division.
171 const float inverted_scale = 1.0f / (*scale);
161 -
162 - scaled_fp8_conversion_vec(out, input, inverted_scale, num_elems, tid,
163 - blockDim.x * gridDim.x);
172 + scaled_fp8_conversion_vec<scalar_t, true>(
173 + out, input, inverted_scale, num_elems, tid, blockDim.x * gridDim.x);
174 }
175
176 template <typename scalar_t>
177 __global__ void dynamic_per_token_scaled_fp8_quant_kernel(
178 c10::Float8_e4m3fn* __restrict__ out, float* __restrict__ scale,
169 - scalar_t const* __restrict__ input, const int hidden_size) {
179 + scalar_t const* __restrict__ input, float const* __restrict__ scale_ub,
180 + const int hidden_size) {
181 + float const min_scaling_factor = 1.0f / (FP8_E4M3_MAX * 512.f);
182 +
183 int const tid = threadIdx.x;
184 int const token_idx = blockIdx.x;
185
@@ -188,20 +201,27 @@ __global__ void dynamic_per_token_scaled_fp8_quant_kernel(
201 }
202
203 float const block_absmax_val_maybe = blockReduceMax(absmax_val);
191 - __shared__ float block_absmax_val;
204 + __shared__ float token_scale;
205 if (tid == 0) {
193 - block_absmax_val = block_absmax_val_maybe;
194 - scale[token_idx] = block_absmax_val / FP8_E4M3_MAX;
206 + if (scale_ub) {
207 + token_scale = min(block_absmax_val_maybe, *scale_ub);
208 + } else {
209 + token_scale = block_absmax_val_maybe;
210 + }
211 + // token scale computation
212 + token_scale = max(token_scale / FP8_E4M3_MAX, min_scaling_factor);
213 + scale[token_idx] = token_scale;
214 }
215 __syncthreads();
216
198 - float const inverted_scale = FP8_E4M3_MAX / block_absmax_val;
217 + // Note that we don't use inverted scales so we can match FBGemm impl.
218 if (can_vectorize) {
200 - scaled_fp8_conversion_vec(token_output, token_input, inverted_scale,
201 - hidden_size, tid, blockDim.x);
219 + scaled_fp8_conversion_vec<scalar_t, false>(
220 + token_output, token_input, token_scale, hidden_size, tid, blockDim.x);
221 } else {
222 for (int i = tid; i < hidden_size; i += blockDim.x) {
204 - token_output[i] = scaled_fp8_conversion(token_input[i], inverted_scale);
223 + token_output[i] = scaled_fp8_conversion<false>(
224 + static_cast<float>(token_input[i]), token_scale);
225 }
226 }
227 }
@@ -246,9 +266,10 @@ void dynamic_scaled_fp8_quant(torch::Tensor& out, // [..., d]
266 });
267 }
268
249 -void dynamic_per_token_scaled_fp8_quant(torch::Tensor& out, // [..., d]
250 - torch::Tensor const& input, // [..., d]
251 - torch::Tensor& scales) {
269 +void dynamic_per_token_scaled_fp8_quant(
270 + torch::Tensor& out, // [..., d]
271 + torch::Tensor const& input, // [..., d]
272 + torch::Tensor& scales, std::optional<at::Tensor> const& scale_ub) {
273 TORCH_CHECK(input.is_contiguous());
274 TORCH_CHECK(out.is_contiguous());
275
@@ -264,6 +285,8 @@ void dynamic_per_token_scaled_fp8_quant(torch::Tensor& out, // [..., d]
285 vllm::dynamic_per_token_scaled_fp8_quant_kernel<scalar_t>
286 <<<grid, block, 0, stream>>>(
287 out.data_ptr<c10::Float8_e4m3fn>(), scales.data_ptr<float>(),
267 - input.data_ptr<scalar_t>(), hidden_size);
288 + input.data_ptr<scalar_t>(),
289 + scale_ub.has_value() ? scale_ub->data_ptr<float>() : nullptr,
290 + hidden_size);
291 });
292 }
csrc/torch_bindings.cpp
+1 -1
@@ -188,7 +188,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
188 // Compute dynamic-per-token FP8 quantized tensor and scaling factor.
189 ops.def(
190 "dynamic_per_token_scaled_fp8_quant(Tensor! out, Tensor input, Tensor! "
191 - "scale) -> "
191 + "scale, Tensor? scale_ub) -> "
192 "()");
193 ops.impl("dynamic_per_token_scaled_fp8_quant", torch::kCUDA,
194 &dynamic_per_token_scaled_fp8_quant);
tests/kernels/quant_utils.py
+23 -7
@@ -1,4 +1,4 @@
1 -from typing import Tuple, Union
1 +from typing import Optional, Tuple, Union
2
3 import torch
4
@@ -7,13 +7,19 @@ def as_float32_tensor(x: Union[float, torch.tensor]) -> torch.tensor:
7 return torch.as_tensor(x, dtype=torch.float32, device='cuda')
8
9 def ref_dynamic_per_token_quant(x: torch.tensor,
10 - quant_dtype: torch.dtype) \
10 + quant_dtype: torch.dtype,
11 + scale_ub: Optional[torch.tensor] = None) \
12 -> Tuple[torch.tensor, torch.tensor]:
13
14 assert quant_dtype in [torch.int8, torch.float8_e4m3fn]
15 + if scale_ub is not None:
16 + assert quant_dtype == torch.float8_e4m3fn
17 +
18 qtype_traits = torch.iinfo(quant_dtype) if quant_dtype == torch.int8 \
19 else torch.finfo(quant_dtype)
20 qtype_max = as_float32_tensor(qtype_traits.max)
21 + s_1 = as_float32_tensor(1.0)
22 + s_512 = as_float32_tensor(512.0)
23
24 # For fp8, in order to match the cuda kernel output, we have to do exactly
25 # the same operations as in the corresponding fp8 kernel to prevent
@@ -22,14 +28,24 @@ def ref_dynamic_per_token_quant(x: torch.tensor,
28 # Compute scales
29 x_token_max, _ = x.abs().max(dim=-1)
30 x_token_max = as_float32_tensor(x_token_max)
31 + if scale_ub is not None:
32 + x_token_max = x_token_max.clamp(max=scale_ub)
33 scales = (x_token_max / qtype_max)[:, None]
34
35 # Quant
28 - iscales = (qtype_max / x_token_max)[:, None]
29 - torch_out = as_float32_tensor(x) * iscales
30 - torch_out = torch_out.round() if quant_dtype == torch.int8 else torch_out
31 - torch_out = torch_out.clamp(qtype_traits.min,
32 - qtype_traits.max).to(quant_dtype)
36 + if quant_dtype == torch.int8:
37 + iscales = as_float32_tensor(s_1 / scales)
38 + torch_out = as_float32_tensor(x) * iscales
39 + torch_out = torch_out.round()
40 + torch_out = torch_out.clamp(qtype_traits.min,
41 + qtype_traits.max).to(quant_dtype)
42 + else:
43 + assert quant_dtype == torch.float8_e4m3fn
44 + min_scaling_factor = s_1 / (qtype_max * s_512)
45 + scales = scales.clamp(min=min_scaling_factor)
46 + torch_out = as_float32_tensor(x) / scales
47 + torch_out = torch_out.clamp(qtype_traits.min,
48 + qtype_traits.max).to(quant_dtype)
49
50 return torch_out, scales
51
tests/kernels/test_fp8_quant.py
+9 -2
@@ -10,24 +10,31 @@ HIDDEN_SIZES = [1, 2, 3, 4, 16, 67, 768, 2048, 5120, 5137, 8192,
10 8193] # Arbitrary values for testing
11 HIDDEN_SIZES += list(range(1024, 1033)) # vectorized conversion edge cases
12 NUM_TOKENS = [1, 7, 83, 4096] # Arbitrary values for testing
13 +SCALE_UBS = [True, False]
14 SEEDS = [0]
15
16
17 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
18 @pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
19 @pytest.mark.parametrize("dtype", DTYPES)
20 +@pytest.mark.parametrize("scale_ub", SCALE_UBS)
21 @pytest.mark.parametrize("seed", SEEDS)
22 @torch.inference_mode()
23 def test_dynamic_per_token_fp8_quant(num_tokens: int, hidden_size: int,
22 - dtype: torch.dtype, seed: int) -> None:
24 + dtype: torch.dtype, scale_ub: bool,
25 + seed: int) -> None:
26 torch.random.manual_seed(seed)
27 torch.cuda.manual_seed(seed)
28
29 x = torch.rand(num_tokens, hidden_size, dtype=dtype,
30 device="cuda") + 1e-6 # avoid nans
31
29 - ref_out, ref_scales = ref_dynamic_per_token_quant(x, torch.float8_e4m3fn)
32 + scale_ub = torch.mean(x).to(dtype=torch.float32, device='cuda') \
33 + if scale_ub else None
34 + ref_out, ref_scales = ref_dynamic_per_token_quant(x, torch.float8_e4m3fn,
35 + scale_ub)
36 ops_out, ops_scales = ops.scaled_fp8_quant(x,
37 + scale_ub=scale_ub,
38 use_per_token_if_dynamic=True)
39
40 assert torch.allclose(ref_scales, ops_scales)
vllm/_custom_ops.py
+2 -1
@@ -300,6 +300,7 @@ def scaled_fp8_quant(
300 input: torch.Tensor,
301 scale: Optional[torch.Tensor] = None,
302 batch_dim_padding: Optional[int] = None,
303 + scale_ub: Optional[torch.Tensor] = None,
304 use_per_token_if_dynamic: bool = False,
305 ) -> Tuple[torch.Tensor, torch.Tensor]:
306 """
@@ -336,7 +337,7 @@ def scaled_fp8_quant(
337 device=input.device,
338 dtype=torch.float32)
339 torch.ops._C.dynamic_per_token_scaled_fp8_quant(
339 - output, input, scale)
340 + output, input, scale, scale_ub)
341 else:
342 scale = torch.zeros(1, device=input.device, dtype=torch.float32)
343 torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)