1
+from typing import Any, Dict, List, Optional
2
+
3
+import torch
4
+from torch.nn import Module
5
+from torch.nn.parameter import Parameter
6
+
7
+from vllm.model_executor.layers.linear import (LinearMethodBase,
8
+ set_weight_attrs)
9
+from vllm.model_executor.layers.quantization.base_config import (
10
+ QuantizationConfig)
11
+
12
+
13
+class FP8Config(QuantizationConfig):
14
+ """Config class for FP8."""
15
+
16
+ @classmethod
17
+ def get_name(cls) -> str:
18
+ return "fp8"
19
+
20
+ @classmethod
21
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
22
+ return [torch.bfloat16, torch.half]
23
+
24
+ @classmethod
25
+ def get_min_capability(cls) -> int:
26
+ # TODO: PyTorch 2.3.0+ is required to run FP8 on
27
+ # SM 89 (e.g. Ada) GPUs. Specifically, this PR has to
28
+ # be included: https://github.com/pytorch/pytorch/pull/118881
29
+ return 90
30
+
31
+ @classmethod
32
+ def get_config_filenames(cls) -> List[str]:
33
+ return []
34
+
35
+ @classmethod
36
+ def from_config(cls, config: Dict[str, Any]) -> "FP8Config":
37
+ return cls()
38
+
39
+ def get_linear_method(self) -> "Fp8LinearMethod":
40
+ return Fp8LinearMethod(self)
41
+
42
+ def get_scaled_act_names(self) -> List[str]:
43
+ return []
44
+
45
+
46
+class Fp8LinearMethod(LinearMethodBase):
47
+ """Linear method for FP8.
48
+ We now support common FP16/BF16 model checkpoints ONLY. The weight
49
+ scaling factor will be initialized after the model weights are loaded.
50
+
51
+ Limitations:
52
+ 1. Only support per-tensor quantization due to torch._scaled_mm support.
53
+ 2. Only support float8_e4m3fn data type due to the limitation of
54
+ torch._scaled_mm (https://github.com/pytorch/pytorch/blob/2e48b39603411a41c5025efbe52f89560b827825/aten/src/ATen/native/cuda/Blas.cpp#L854-L856)
55
+
56
+ Args:
57
+ quant_config: The quantization config.
58
+ """
59
+
60
+ def __init__(self, quant_config: FP8Config):
61
+ self.quant_config = quant_config
62
+
63
+ def create_weights(
64
+ self,
65
+ layer: torch.nn.Module,
66
+ input_size_per_partition: int,
67
+ output_size_per_partition: int,
68
+ input_size: int,
69
+ output_size: int,
70
+ params_dtype: torch.dtype,
71
+ **extra_weight_attrs,
72
+ ):
73
+ weight = Parameter(torch.empty(output_size_per_partition,
74
+ input_size_per_partition,
75
+ dtype=params_dtype),
76
+ requires_grad=False)
77
+ layer.register_parameter("weight", weight)
78
+ set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
79
+ set_weight_attrs(weight, extra_weight_attrs)
80
+
81
+ w_scale = Parameter(
82
+ torch.empty(1, dtype=torch.float32),
83
+ requires_grad=False,
84
+ )
85
+ layer.register_parameter("weight_scaling_factor", w_scale)
86
+
87
+ def process_weights_after_loading(self, layer: Module) -> None:
88
+ # Although the linear_method is propagated to all layers,
89
+ # only linear layers invoke "create_weights". So we check
90
+ # whether "weight_scaling_facor" is registered to determine
91
+ # whether the layer is a linear layer that requires quantization.
92
+ if not hasattr(layer, "weight_scaling_factor"):
93
+ return
94
+
95
+ qweight, weight_scale = per_tensor_quantize(layer.weight)
96
+ # torch._scaled_mm requires column-major in the second
97
+ # input (weight), so we transpose the quantized weight.
98
+ layer.weight = Parameter(qweight.t(), requires_grad=False)
99
+ layer.weight_scaling_factor.data.copy_(weight_scale)
100
+
101
+ def apply_weights(self,
102
+ layer: torch.nn.Module,
103
+ x: torch.Tensor,
104
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
105
+ qinput, x_scale = per_tensor_quantize(x)
106
+ output, _ = torch._scaled_mm(
107
+ qinput,
108
+ layer.weight,
109
+ out_dtype=x.dtype,
110
+ scale_a=x_scale,
111
+ scale_b=layer.weight_scaling_factor,
112
+ bias=bias,
113
+ )
114
+ return output
115
+
116
+
117
+def per_tensor_quantize(tensor: torch.Tensor) -> tuple[torch.Tensor, float]:
118
+ """Quantize a tensor using per-tensor static scaling factor.
119
+
120
+ Args:
121
+ tensor: The input tensor.
122
+ """
123
+ finfo = torch.finfo(torch.float8_e4m3fn)
124
+ # Calculate the scale as dtype max divided by absmax.
125
+ # Since .abs() creates a new tensor, we use aminmax to get
126
+ # the min and max first and then calculate the absmax.
127
+ min_val, max_val = tensor.aminmax()
128
+ amax = min_val.abs().max(max_val.abs())
129
+ scale = finfo.max / amax.clamp(min=1e-12)
130
+ # scale and clamp the tensor to bring it to
131
+ # the representative range of float8 data type
132
+ # (as default cast is unsaturated)
133
+ qweight = (tensor * scale).clamp(min=finfo.min, max=finfo.max)
134
+ # Return both float8 data and the inverse scale (as float),
135
+ # as both required as inputs to torch._scaled_mm
136
+ qweight = qweight.to(torch.float8_e4m3fn)
137
+ scale = scale.float().reciprocal()
138
+ return qweight, scale