A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[ Kernel ] Enable `fp8-marlin` for `fbgemm-fp8` models (#6606)
Robert Shaw committed
Jul 20, 2024 at 14:50 UTC
9364f74eee2e8aab9e3c9cd6dea290018ef43b95
4 files changed
+44
-3
.buildkite/lm-eval-harness/configs/Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m nm-testing/Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform -b auto -l 1000 -f 5
2
+model_name: "nm-testing/Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.905
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.905
10
+limit: 1000
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-large.txt
+1
@@ -1,3 +1,4 @@
1
+Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform.yaml
2
Meta-Llama-3-70B-Instruct.yaml
3
Mixtral-8x7B-Instruct-v0.1.yaml
4
Qwen2-57B-A14-Instruct.yaml
vllm/model_executor/layers/quantization/fbgemm_fp8.py
+25
-1
@@ -9,9 +9,12 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
9
UnquantizedLinearMethod)
10
from vllm.model_executor.layers.quantization.base_config import (
11
QuantizationConfig, QuantizeMethodBase)
12
+from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
13
+ apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
14
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
15
apply_fp8_linear, create_per_channel_scale_param)
16
from vllm.model_executor.utils import set_weight_attrs
17
+from vllm.platforms import current_platform
18
19
logger = init_logger(__name__)
20
@@ -31,6 +34,12 @@ class FBGEMMFp8Config(QuantizationConfig):
34
self.ignore_list = ignore_list
35
self.input_scale_ub = input_scale_ub
36
37
+ # For GPUs that lack FP8 hardware support, we can leverage the Marlin
38
+ # kernel for fast weight-only FP8 quantization
39
+ capability = current_platform.get_device_capability()
40
+ capability = capability[0] * 10 + capability[1]
41
+ self.use_marlin = capability < 89
42
+
43
@classmethod
44
def get_name(cls) -> str:
45
return "fbgemm_fp8"
@@ -41,7 +50,7 @@ class FBGEMMFp8Config(QuantizationConfig):
50
51
@classmethod
52
def get_min_capability(cls) -> int:
44
- return 89
53
+ return 80
54
55
@classmethod
56
def get_config_filenames(cls) -> List[str]:
@@ -143,11 +152,26 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
152
weight = layer.weight
153
layer.weight = Parameter(weight.t(), requires_grad=False)
154
155
+ if self.quant_config.use_marlin:
156
+ prepare_fp8_layer_for_marlin(layer)
157
+ # Activations not quantized for marlin.
158
+ del layer.input_scale_ub
159
+
160
def apply(self,
161
layer: torch.nn.Module,
162
x: torch.Tensor,
163
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
164
165
+ if self.quant_config.use_marlin:
166
+ return apply_fp8_marlin_linear(
167
+ input=x,
168
+ weight=layer.weight,
169
+ weight_scale=layer.weight_scale,
170
+ workspace=layer.workspace,
171
+ size_n=layer.output_size_per_partition,
172
+ size_k=layer.input_size_per_partition,
173
+ bias=bias)
174
+
175
return apply_fp8_linear(input=x,
176
weight=layer.weight,
177
weight_scale=layer.weight_scale,
vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py
+7
-2
@@ -76,8 +76,13 @@ def prepare_fp8_layer_for_marlin(layer: torch.nn.Module) -> None:
76
# WEIGHT SCALES
77
# Currently Marlin doesn't support per-tensor scales, so we
78
# expand it to channelwise
79
- scales = layer.weight_scale.repeat(1, part_size_n).to(
80
- layer.orig_dtype).to(device)
79
+ is_channelwise = layer.weight_scale.shape[0] == part_size_n
80
+ if is_channelwise:
81
+ scales = layer.weight_scale
82
+ else:
83
+ scales = layer.weight_scale.repeat(1, part_size_n)
84
+ scales = scales.to(layer.orig_dtype).to(device)
85
+
86
# Permute scales
87
marlin_scales = marlin_permute_scales(s=scales,
88
size_k=part_size_k,