A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Kernel] Remove scaled_fp8_quant kernel padding footgun (#6842)
Tyler Michael Smith committed
Jul 30, 2024 at 16:37 UTC
d7a299edaa5d23f3d7d5c98b53872a8ced9aad80
3 files changed
+17
-14
tests/quantization/test_fp8.py
+1
-1
@@ -123,7 +123,7 @@ def test_scaled_fp8_quant(dtype) -> None:
123
assert torch.allclose(ref_y, per_tensor_dequantize(y, inv_scale, dtype))
124
125
# Padding
126
- y, _ = ops.scaled_fp8_quant(x, inv_scale, batch_dim_padding=17)
126
+ y, _ = ops.scaled_fp8_quant(x, inv_scale, num_token_padding=17)
127
assert y.shape[0] == 17
128
assert torch.allclose(
129
ref_y,
vllm/_custom_ops.py
+13
-11
@@ -307,7 +307,7 @@ def fp8_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
307
def scaled_fp8_quant(
308
input: torch.Tensor,
309
scale: Optional[torch.Tensor] = None,
310
- batch_dim_padding: Optional[int] = None,
310
+ num_token_padding: Optional[int] = None,
311
scale_ub: Optional[torch.Tensor] = None,
312
use_per_token_if_dynamic: bool = False,
313
) -> Tuple[torch.Tensor, torch.Tensor]:
@@ -317,7 +317,7 @@ def scaled_fp8_quant(
317
This function supports both static and dynamic quantization: If you
318
provide the scale, it will use static scaling and if you omit it,
319
the scale will be determined dynamically. The function also allows
320
- optional padding of the output tensor for downstream kernels that
320
+ optional padding of the output tensors for downstream kernels that
321
will benefit from padding.
322
323
Args:
@@ -325,7 +325,7 @@ def scaled_fp8_quant(
325
scale: Optional scaling factor for the FP8 quantization
326
scale_ub: Optional upper bound for scaling factor in dynamic
327
per token case
328
- batch_dim_padding: If specified, pad the first dimension
328
+ num_token_padding: If specified, pad the first dimension
329
of the output to at least this value.
330
use_per_token_if_dynamic: Whether to do per_tensor or per_token
331
in the dynamic quantization case.
@@ -334,16 +334,16 @@ def scaled_fp8_quant(
334
Tuple[torch.Tensor, torch.Tensor]: The output tensor in FP8 and
335
scaling factor.
336
"""
337
- if batch_dim_padding:
338
- shape = (max(batch_dim_padding, input.shape[0]), *input.shape[1:])
339
- output = torch.empty(shape,
340
- device=input.device,
341
- dtype=torch.float8_e4m3fn)
342
- else:
343
- output = torch.empty_like(input, dtype=torch.float8_e4m3fn)
337
+ # This code assumes batch_dim and num_tokens are flattened
338
+ assert (input.ndim == 2)
339
+ shape = input.shape
340
+ if num_token_padding:
341
+ shape = (max(num_token_padding, input.shape[0]), shape[1])
342
+ output = torch.empty(shape, device=input.device, dtype=torch.float8_e4m3fn)
343
+
344
if scale is None:
345
if use_per_token_if_dynamic:
346
- scale = torch.empty((input.numel() // input.shape[-1], 1),
346
+ scale = torch.empty((shape[0], 1),
347
device=input.device,
348
dtype=torch.float32)
349
torch.ops._C.dynamic_per_token_scaled_fp8_quant(
@@ -352,6 +352,8 @@ def scaled_fp8_quant(
352
scale = torch.zeros(1, device=input.device, dtype=torch.float32)
353
torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)
354
else:
355
+ # num_token_padding not implemented for this case
356
+ assert (scale.numel() == 1 or num_token_padding is None)
357
torch.ops._C.static_scaled_fp8_quant(output, input, scale)
358
359
return output, scale
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+3
-2
@@ -139,7 +139,7 @@ def apply_fp8_linear(
139
qinput, x_scale = ops.scaled_fp8_quant(
140
input,
141
input_scale,
142
- batch_dim_padding=17,
142
+ num_token_padding=17,
143
use_per_token_if_dynamic=use_per_token_if_dynamic)
144
145
per_tensor_weights = (weight_scale.numel() == 1)
@@ -177,8 +177,9 @@ def apply_fp8_linear(
177
output, _ = torch._scaled_mm(qinput,
178
weight,
179
out_dtype=torch.float32)
180
- # Unpad (undo batch_dim_padding)
180
+ # Unpad (undo num_token_padding)
181
output = torch.narrow(output, 0, 0, input.shape[0])
182
+ x_scale = torch.narrow(x_scale, 0, 0, input.shape[0])
183
184
# DQ
185
# C = sw * sx * (X * W) + bias