@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a22cdea371
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Kernel][FP8] Initial support with dynamic per-tensor scaling (#4118)

Provide an initial support to FP8 computation. This PR is inspired by HuggingFace TGI: huggingface/text-generation-inference#1726 This feature can be enabled with --quantization fp8 or -q fp8 when launching an engine. Algorithm: We still load a model checkpoint in FP16/BF16. After the weights are loaded, Fp8LinearMethod calculates the per-tensor scaling factor of weights and quantizes the weights accordingly. The scaling factor will then be stored for future use. Meanwhile, the per-tensor scaling factor for activations is calculated in every forward pass. Initial Results: Currently tested Mistral-7B on 1xH100. With prompt length ~5 and decoding length 128: BF16: 1.47s FP8: 1.66s I'll try to use larger models and try to find more performance bottleneck. Meanwhile, you're welcome to try this code.

Cody Yu committed Apr 19, 2024 at 21:28 UTC a22cdea371bb26b4bdba112d4602736b48ca4a3a
7 files changed +189 -5
tests/quantization/test_fp8.py new
+24
@@ -0,0 +1,24 @@
1 +"""Tests whether FP8 computation is enabled correctly.
2 +
3 +Run `pytest tests/quantization/test_fp8.py --forked`.
4 +"""
5 +import pytest
6 +import torch
7 +
8 +from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
9 +from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
10 +
11 +capability = torch.cuda.get_device_capability()
12 +capability = capability[0] * 10 + capability[1]
13 +
14 +
15 +@pytest.mark.skipif(
16 + capability < QUANTIZATION_METHODS["fp8"].get_min_capability(),
17 + reason="FP8 is not supported on this GPU type.")
18 +def test_load_fp16_model(vllm_runner) -> None:
19 + llm = vllm_runner("facebook/opt-125m", quantization="fp8")
20 +
21 + model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model
22 + fc1 = model.model.decoder.layers[0].fc1
23 + assert isinstance(fc1.linear_method, Fp8LinearMethod)
24 + assert fc1.weight.dtype == torch.float8_e4m3fn
vllm/entrypoints/llm.py
+5 -4
@@ -42,10 +42,11 @@ class LLM:
42 However, if the `torch_dtype` in the config is `float32`, we will
43 use `float16` instead.
44 quantization: The method used to quantize the model weights. Currently,
45 - we support "awq", "gptq" and "squeezellm". If None, we first check
46 - the `quantization_config` attribute in the model config file. If
47 - that is None, we assume the model weights are not quantized and use
48 - `dtype` to determine the data type of the weights.
45 + we support "awq", "gptq", "squeezellm", and "fp8" (experimental).
46 + If None, we first check the `quantization_config` attribute in the
47 + model config file. If that is None, we assume the model weights are
48 + not quantized and use `dtype` to determine the data type of
49 + the weights.
50 revision: The specific model version to use. It can be a branch name,
51 a tag name, or a commit id.
52 tokenizer_revision: The specific tokenizer version to use. It can be a
vllm/model_executor/layers/linear.py
+8
@@ -3,6 +3,7 @@ from typing import List, Optional
3
4 import torch
5 import torch.nn.functional as F
6 +from torch import nn
7 from torch.nn.parameter import Parameter
8
9 from vllm.distributed import (divide, get_tensor_model_parallel_rank,
@@ -48,6 +49,13 @@ class LinearMethodBase(ABC):
49 Expects create_weights to have been called before on the layer."""
50 raise NotImplementedError
51
52 + def process_weights_after_loading(self, layer: nn.Module) -> None:
53 + """Process the weight after loading.
54 +
55 + This can be used for example, to transpose weights for computation.
56 + """
57 + return
58 +
59
60 class UnquantizedLinearMethod(LinearMethodBase):
61 """Linear method without quantization.
vllm/model_executor/layers/quantization/__init__.py
+2
@@ -3,12 +3,14 @@ from typing import Type
3 from vllm.model_executor.layers.quantization.awq import AWQConfig
4 from vllm.model_executor.layers.quantization.base_config import (
5 QuantizationConfig)
6 +from vllm.model_executor.layers.quantization.fp8 import FP8Config
7 from vllm.model_executor.layers.quantization.gptq import GPTQConfig
8 from vllm.model_executor.layers.quantization.marlin import MarlinConfig
9 from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
10
11 QUANTIZATION_METHODS = {
12 "awq": AWQConfig,
13 + "fp8": FP8Config,
14 "gptq": GPTQConfig,
15 "squeezellm": SqueezeLLMConfig,
16 "marlin": MarlinConfig,
vllm/model_executor/layers/quantization/fp8.py new
+138
@@ -0,0 +1,138 @@
1 +from typing import Any, Dict, List, Optional
2 +
3 +import torch
4 +from torch.nn import Module
5 +from torch.nn.parameter import Parameter
6 +
7 +from vllm.model_executor.layers.linear import (LinearMethodBase,
8 + set_weight_attrs)
9 +from vllm.model_executor.layers.quantization.base_config import (
10 + QuantizationConfig)
11 +
12 +
13 +class FP8Config(QuantizationConfig):
14 + """Config class for FP8."""
15 +
16 + @classmethod
17 + def get_name(cls) -> str:
18 + return "fp8"
19 +
20 + @classmethod
21 + def get_supported_act_dtypes(cls) -> List[torch.dtype]:
22 + return [torch.bfloat16, torch.half]
23 +
24 + @classmethod
25 + def get_min_capability(cls) -> int:
26 + # TODO: PyTorch 2.3.0+ is required to run FP8 on
27 + # SM 89 (e.g. Ada) GPUs. Specifically, this PR has to
28 + # be included: https://github.com/pytorch/pytorch/pull/118881
29 + return 90
30 +
31 + @classmethod
32 + def get_config_filenames(cls) -> List[str]:
33 + return []
34 +
35 + @classmethod
36 + def from_config(cls, config: Dict[str, Any]) -> "FP8Config":
37 + return cls()
38 +
39 + def get_linear_method(self) -> "Fp8LinearMethod":
40 + return Fp8LinearMethod(self)
41 +
42 + def get_scaled_act_names(self) -> List[str]:
43 + return []
44 +
45 +
46 +class Fp8LinearMethod(LinearMethodBase):
47 + """Linear method for FP8.
48 + We now support common FP16/BF16 model checkpoints ONLY. The weight
49 + scaling factor will be initialized after the model weights are loaded.
50 +
51 + Limitations:
52 + 1. Only support per-tensor quantization due to torch._scaled_mm support.
53 + 2. Only support float8_e4m3fn data type due to the limitation of
54 + torch._scaled_mm (https://github.com/pytorch/pytorch/blob/2e48b39603411a41c5025efbe52f89560b827825/aten/src/ATen/native/cuda/Blas.cpp#L854-L856)
55 +
56 + Args:
57 + quant_config: The quantization config.
58 + """
59 +
60 + def __init__(self, quant_config: FP8Config):
61 + self.quant_config = quant_config
62 +
63 + def create_weights(
64 + self,
65 + layer: torch.nn.Module,
66 + input_size_per_partition: int,
67 + output_size_per_partition: int,
68 + input_size: int,
69 + output_size: int,
70 + params_dtype: torch.dtype,
71 + **extra_weight_attrs,
72 + ):
73 + weight = Parameter(torch.empty(output_size_per_partition,
74 + input_size_per_partition,
75 + dtype=params_dtype),
76 + requires_grad=False)
77 + layer.register_parameter("weight", weight)
78 + set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
79 + set_weight_attrs(weight, extra_weight_attrs)
80 +
81 + w_scale = Parameter(
82 + torch.empty(1, dtype=torch.float32),
83 + requires_grad=False,
84 + )
85 + layer.register_parameter("weight_scaling_factor", w_scale)
86 +
87 + def process_weights_after_loading(self, layer: Module) -> None:
88 + # Although the linear_method is propagated to all layers,
89 + # only linear layers invoke "create_weights". So we check
90 + # whether "weight_scaling_facor" is registered to determine
91 + # whether the layer is a linear layer that requires quantization.
92 + if not hasattr(layer, "weight_scaling_factor"):
93 + return
94 +
95 + qweight, weight_scale = per_tensor_quantize(layer.weight)
96 + # torch._scaled_mm requires column-major in the second
97 + # input (weight), so we transpose the quantized weight.
98 + layer.weight = Parameter(qweight.t(), requires_grad=False)
99 + layer.weight_scaling_factor.data.copy_(weight_scale)
100 +
101 + def apply_weights(self,
102 + layer: torch.nn.Module,
103 + x: torch.Tensor,
104 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
105 + qinput, x_scale = per_tensor_quantize(x)
106 + output, _ = torch._scaled_mm(
107 + qinput,
108 + layer.weight,
109 + out_dtype=x.dtype,
110 + scale_a=x_scale,
111 + scale_b=layer.weight_scaling_factor,
112 + bias=bias,
113 + )
114 + return output
115 +
116 +
117 +def per_tensor_quantize(tensor: torch.Tensor) -> tuple[torch.Tensor, float]:
118 + """Quantize a tensor using per-tensor static scaling factor.
119 +
120 + Args:
121 + tensor: The input tensor.
122 + """
123 + finfo = torch.finfo(torch.float8_e4m3fn)
124 + # Calculate the scale as dtype max divided by absmax.
125 + # Since .abs() creates a new tensor, we use aminmax to get
126 + # the min and max first and then calculate the absmax.
127 + min_val, max_val = tensor.aminmax()
128 + amax = min_val.abs().max(max_val.abs())
129 + scale = finfo.max / amax.clamp(min=1e-12)
130 + # scale and clamp the tensor to bring it to
131 + # the representative range of float8 data type
132 + # (as default cast is unsaturated)
133 + qweight = (tensor * scale).clamp(min=finfo.min, max=finfo.max)
134 + # Return both float8 data and the inverse scale (as float),
135 + # as both required as inputs to torch._scaled_mm
136 + qweight = qweight.to(torch.float8_e4m3fn)
137 + scale = scale.float().reciprocal()
138 + return qweight, scale
vllm/model_executor/model_loader/loader.py
+4
@@ -228,6 +228,10 @@ class DefaultModelLoader(BaseModelLoader):
228 model,
229 "fall_back_to_pt_during_load",
230 True)), )
231 + for _, module in model.named_modules():
232 + linear_method = getattr(module, "linear_method", None)
233 + if linear_method is not None:
234 + linear_method.process_weights_after_loading(module)
235 return model.eval()
236
237
vllm/model_executor/model_loader/weight_utils.py
+8 -1
@@ -134,11 +134,18 @@ def get_quant_config(model_config: ModelConfig,
134 tqdm_class=DisabledTqdm)
135 else:
136 hf_folder = model_name_or_path
137 +
138 + possible_config_filenames = quant_cls.get_config_filenames()
139 +
140 + # If the quantization config is not found, use the default config.
141 + if not possible_config_filenames:
142 + return quant_cls()
143 +
144 config_files = glob.glob(os.path.join(hf_folder, "*.json"))
145
146 quant_config_files = [
147 f for f in config_files if any(
141 - f.endswith(x) for x in quant_cls.get_config_filenames())
148 + f.endswith(x) for x in possible_config_filenames)
149 ]
150 if len(quant_config_files) == 0:
151 raise ValueError(