A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[AMD][FP8] Using MI300 FP8 format on ROCm for block_quant (#12134)
Signed-off-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com>
Gregory Shtrasberg committed
Jan 17, 2025 at 12:12 UTC
b5b57e301e7bce3a90af7a3ed206414c46eb64e0
2 files changed
+44
-3
vllm/model_executor/layers/quantization/fp8.py
+33
@@ -247,6 +247,15 @@ class Fp8LinearMethod(LinearMethodBase):
247
def process_weights_after_loading(self, layer: Module) -> None:
248
# Block quant doesn't need to process weights after loading
249
if self.block_quant:
250
+ if current_platform.is_rocm():
251
+ weight, weight_scale, _ = \
252
+ normalize_e4m3fn_to_e4m3fnuz(
253
+ weight=layer.weight,
254
+ weight_scale=layer.weight_scale_inv,
255
+ input_scale=layer.input_scale)
256
+ layer.weight = Parameter(weight, requires_grad=False)
257
+ layer.weight_scale_inv = Parameter(weight_scale,
258
+ requires_grad=False)
259
return
260
layer.weight = torch.nn.Parameter(layer.weight.data,
261
requires_grad=False)
@@ -495,6 +504,30 @@ class Fp8MoEMethod(FusedMoEMethodBase):
504
def process_weights_after_loading(self, layer: Module) -> None:
505
# Block quant doesn't need to process weights after loading
506
if self.block_quant:
507
+ if current_platform.is_rocm():
508
+ w13_weight, w13_weight_scale_inv, w13_input_scale = \
509
+ normalize_e4m3fn_to_e4m3fnuz(
510
+ layer.w13_weight, layer.w13_weight_scale_inv,
511
+ layer.w13_input_scale)
512
+ w2_weight, w2_weight_scale_inv, w2_input_scale = \
513
+ normalize_e4m3fn_to_e4m3fnuz(
514
+ layer.w2_weight, layer.w2_weight_scale_inv,
515
+ layer.w2_input_scale)
516
+ # Reset the parameter
517
+ layer.w13_weight = torch.nn.Parameter(w13_weight,
518
+ requires_grad=False)
519
+ layer.w13_weight_scale_inv = torch.nn.Parameter(
520
+ w13_weight_scale_inv, requires_grad=False)
521
+ if w13_input_scale is not None:
522
+ layer.w13_input_scale = torch.nn.Parameter(
523
+ w13_input_scale, requires_grad=False)
524
+ layer.w2_weight = torch.nn.Parameter(w2_weight,
525
+ requires_grad=False)
526
+ layer.w2_weight_scale_inv = torch.nn.Parameter(
527
+ w2_weight_scale_inv, requires_grad=False)
528
+ if w2_input_scale is not None:
529
+ layer.w2_input_scale = torch.nn.Parameter(
530
+ w2_input_scale, requires_grad=False)
531
return
532
# If checkpoint is fp16, quantize in place.
533
if not self.quant_config.is_checkpoint_fp8_serialized:
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+11
-3
@@ -5,6 +5,8 @@ import torch
5
import triton
6
import triton.language as tl
7
8
+from vllm.platforms import current_platform
9
+
10
11
def apply_w8a8_block_fp8_linear(
12
input: torch.Tensor,
@@ -33,11 +35,14 @@ def apply_w8a8_block_fp8_linear(
35
36
37
def input_to_float8(
36
- x: torch.Tensor,
37
- dtype: torch.dtype = torch.float8_e4m3fn
38
+ x: torch.Tensor,
39
+ dtype: Optional[torch.dtype] = None
40
) -> Tuple[torch.Tensor, torch.Tensor]:
41
"""This function quantizes input values to float8 values "
42
"with tensor-wise quantization."""
43
+ if dtype is None:
44
+ dtype = (torch.float8_e4m3fnuz
45
+ if current_platform.is_rocm() else torch.float8_e4m3fn)
46
finfo = torch.finfo(dtype)
47
min_val, max_val = x.aminmax()
48
amax = torch.maximum(min_val.abs(), max_val.abs()).clamp(min=1e-12)
@@ -125,7 +130,7 @@ def per_token_group_quant_fp8(
130
x: torch.Tensor,
131
group_size: int,
132
eps: float = 1e-10,
128
- dtype: torch.dtype = torch.float8_e4m3fn,
133
+ dtype: Optional[torch.dtype] = None,
134
) -> Tuple[torch.Tensor, torch.Tensor]:
135
"""Function to perform per-token-group quantization on an input tensor `x`.
136
It converts the tensor values into signed float8 values and returns the
@@ -140,6 +145,9 @@ def per_token_group_quant_fp8(
145
Tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the
146
scaling factor for quantization.
147
"""
148
+ if dtype is None:
149
+ dtype = (torch.float8_e4m3fnuz
150
+ if current_platform.is_rocm() else torch.float8_e4m3fn)
151
assert (x.shape[-1] % group_size == 0), (
152
f"the last dimension of `x` {x.shape[-1]} must be divisible "
153
f"by `group_size` {group_size}")