@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d7a299edaa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Kernel] Remove scaled_fp8_quant kernel padding footgun (#6842)

Tyler Michael Smith committed Jul 30, 2024 at 16:37 UTC d7a299edaa5d23f3d7d5c98b53872a8ced9aad80
3 files changed +17 -14
tests/quantization/test_fp8.py
+1 -1
@@ -123,7 +123,7 @@ def test_scaled_fp8_quant(dtype) -> None:
123 assert torch.allclose(ref_y, per_tensor_dequantize(y, inv_scale, dtype))
124
125 # Padding
126 - y, _ = ops.scaled_fp8_quant(x, inv_scale, batch_dim_padding=17)
126 + y, _ = ops.scaled_fp8_quant(x, inv_scale, num_token_padding=17)
127 assert y.shape[0] == 17
128 assert torch.allclose(
129 ref_y,
vllm/_custom_ops.py
+13 -11
@@ -307,7 +307,7 @@ def fp8_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
307 def scaled_fp8_quant(
308 input: torch.Tensor,
309 scale: Optional[torch.Tensor] = None,
310 - batch_dim_padding: Optional[int] = None,
310 + num_token_padding: Optional[int] = None,
311 scale_ub: Optional[torch.Tensor] = None,
312 use_per_token_if_dynamic: bool = False,
313 ) -> Tuple[torch.Tensor, torch.Tensor]:
@@ -317,7 +317,7 @@ def scaled_fp8_quant(
317 This function supports both static and dynamic quantization: If you
318 provide the scale, it will use static scaling and if you omit it,
319 the scale will be determined dynamically. The function also allows
320 - optional padding of the output tensor for downstream kernels that
320 + optional padding of the output tensors for downstream kernels that
321 will benefit from padding.
322
323 Args:
@@ -325,7 +325,7 @@ def scaled_fp8_quant(
325 scale: Optional scaling factor for the FP8 quantization
326 scale_ub: Optional upper bound for scaling factor in dynamic
327 per token case
328 - batch_dim_padding: If specified, pad the first dimension
328 + num_token_padding: If specified, pad the first dimension
329 of the output to at least this value.
330 use_per_token_if_dynamic: Whether to do per_tensor or per_token
331 in the dynamic quantization case.
@@ -334,16 +334,16 @@ def scaled_fp8_quant(
334 Tuple[torch.Tensor, torch.Tensor]: The output tensor in FP8 and
335 scaling factor.
336 """
337 - if batch_dim_padding:
338 - shape = (max(batch_dim_padding, input.shape[0]), *input.shape[1:])
339 - output = torch.empty(shape,
340 - device=input.device,
341 - dtype=torch.float8_e4m3fn)
342 - else:
343 - output = torch.empty_like(input, dtype=torch.float8_e4m3fn)
337 + # This code assumes batch_dim and num_tokens are flattened
338 + assert (input.ndim == 2)
339 + shape = input.shape
340 + if num_token_padding:
341 + shape = (max(num_token_padding, input.shape[0]), shape[1])
342 + output = torch.empty(shape, device=input.device, dtype=torch.float8_e4m3fn)
343 +
344 if scale is None:
345 if use_per_token_if_dynamic:
346 - scale = torch.empty((input.numel() // input.shape[-1], 1),
346 + scale = torch.empty((shape[0], 1),
347 device=input.device,
348 dtype=torch.float32)
349 torch.ops._C.dynamic_per_token_scaled_fp8_quant(
@@ -352,6 +352,8 @@ def scaled_fp8_quant(
352 scale = torch.zeros(1, device=input.device, dtype=torch.float32)
353 torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)
354 else:
355 + # num_token_padding not implemented for this case
356 + assert (scale.numel() == 1 or num_token_padding is None)
357 torch.ops._C.static_scaled_fp8_quant(output, input, scale)
358
359 return output, scale
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+3 -2
@@ -139,7 +139,7 @@ def apply_fp8_linear(
139 qinput, x_scale = ops.scaled_fp8_quant(
140 input,
141 input_scale,
142 - batch_dim_padding=17,
142 + num_token_padding=17,
143 use_per_token_if_dynamic=use_per_token_if_dynamic)
144
145 per_tensor_weights = (weight_scale.numel() == 1)
@@ -177,8 +177,9 @@ def apply_fp8_linear(
177 output, _ = torch._scaled_mm(qinput,
178 weight,
179 out_dtype=torch.float32)
180 - # Unpad (undo batch_dim_padding)
180 + # Unpad (undo num_token_padding)
181 output = torch.narrow(output, 0, 0, input.shape[0])
182 + x_scale = torch.narrow(x_scale, 0, 0, input.shape[0])
183
184 # DQ
185 # C = sw * sx * (X * W) + bias