@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9364f74eee
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[ Kernel ] Enable `fp8-marlin` for `fbgemm-fp8` models (#6606)

Robert Shaw committed Jul 20, 2024 at 14:50 UTC 9364f74eee2e8aab9e3c9cd6dea290018ef43b95
4 files changed +44 -3
.buildkite/lm-eval-harness/configs/Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m nm-testing/Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform -b auto -l 1000 -f 5
2 +model_name: "nm-testing/Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.905
8 + - name: "exact_match,flexible-extract"
9 + value: 0.905
10 +limit: 1000
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-large.txt
+1
@@ -1,3 +1,4 @@
1 +Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform.yaml
2 Meta-Llama-3-70B-Instruct.yaml
3 Mixtral-8x7B-Instruct-v0.1.yaml
4 Qwen2-57B-A14-Instruct.yaml
vllm/model_executor/layers/quantization/fbgemm_fp8.py
+25 -1
@@ -9,9 +9,12 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
9 UnquantizedLinearMethod)
10 from vllm.model_executor.layers.quantization.base_config import (
11 QuantizationConfig, QuantizeMethodBase)
12 +from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
13 + apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
14 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
15 apply_fp8_linear, create_per_channel_scale_param)
16 from vllm.model_executor.utils import set_weight_attrs
17 +from vllm.platforms import current_platform
18
19 logger = init_logger(__name__)
20
@@ -31,6 +34,12 @@ class FBGEMMFp8Config(QuantizationConfig):
34 self.ignore_list = ignore_list
35 self.input_scale_ub = input_scale_ub
36
37 + # For GPUs that lack FP8 hardware support, we can leverage the Marlin
38 + # kernel for fast weight-only FP8 quantization
39 + capability = current_platform.get_device_capability()
40 + capability = capability[0] * 10 + capability[1]
41 + self.use_marlin = capability < 89
42 +
43 @classmethod
44 def get_name(cls) -> str:
45 return "fbgemm_fp8"
@@ -41,7 +50,7 @@ class FBGEMMFp8Config(QuantizationConfig):
50
51 @classmethod
52 def get_min_capability(cls) -> int:
44 - return 89
53 + return 80
54
55 @classmethod
56 def get_config_filenames(cls) -> List[str]:
@@ -143,11 +152,26 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
152 weight = layer.weight
153 layer.weight = Parameter(weight.t(), requires_grad=False)
154
155 + if self.quant_config.use_marlin:
156 + prepare_fp8_layer_for_marlin(layer)
157 + # Activations not quantized for marlin.
158 + del layer.input_scale_ub
159 +
160 def apply(self,
161 layer: torch.nn.Module,
162 x: torch.Tensor,
163 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
164
165 + if self.quant_config.use_marlin:
166 + return apply_fp8_marlin_linear(
167 + input=x,
168 + weight=layer.weight,
169 + weight_scale=layer.weight_scale,
170 + workspace=layer.workspace,
171 + size_n=layer.output_size_per_partition,
172 + size_k=layer.input_size_per_partition,
173 + bias=bias)
174 +
175 return apply_fp8_linear(input=x,
176 weight=layer.weight,
177 weight_scale=layer.weight_scale,
vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py
+7 -2
@@ -76,8 +76,13 @@ def prepare_fp8_layer_for_marlin(layer: torch.nn.Module) -> None:
76 # WEIGHT SCALES
77 # Currently Marlin doesn't support per-tensor scales, so we
78 # expand it to channelwise
79 - scales = layer.weight_scale.repeat(1, part_size_n).to(
80 - layer.orig_dtype).to(device)
79 + is_channelwise = layer.weight_scale.shape[0] == part_size_n
80 + if is_channelwise:
81 + scales = layer.weight_scale
82 + else:
83 + scales = layer.weight_scale.repeat(1, part_size_n)
84 + scales = scales.to(layer.orig_dtype).to(device)
85 +
86 # Permute scales
87 marlin_scales = marlin_permute_scales(s=scales,
88 size_k=part_size_k,