@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0eb0757bef
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Misc] Add ignored layers for `fp8` quantization (#6657)

Michael Goin committed Jul 23, 2024 at 14:04 UTC 0eb0757bef728c632be17e61a88cfa9dc1a760c4
4 files changed +57 -47
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+5 -9
@@ -5,6 +5,9 @@ from typing import Any, Dict, Iterable, Optional
5 from pydantic import BaseModel, Field
6 from torch.nn import Module
7
8 +from vllm.model_executor.layers.quantization.utils.quant_utils import (
9 + FUSED_LAYER_NAME_MAPPING)
10 +
11
12 class CompressionFormat(Enum):
13 dense = "dense"
@@ -86,13 +89,6 @@ def is_activation_quantization_format(format: str) -> bool:
89 return format in _ACTIVATION_QUANTIZATION_FORMATS
90
91
89 -# fused_name: List[shard_name]
90 -_FUSED_LAYER_NAME_MAPPING = {
91 - "qkv_proj": ["q_proj", "k_proj", "v_proj"],
92 - "gate_up_proj": ["gate_proj", "up_proj"]
93 -}
94 -
95 -
92 def should_ignore_layer(layer_name: Optional[str],
93 ignore: Iterable[str]) -> bool:
94 if layer_name is None:
@@ -106,8 +102,8 @@ def should_ignore_layer(layer_name: Optional[str],
102 # in the safetensors checkpoint. So, we convert the name
103 # from the fused version to unfused + check to make sure that
104 # each shard of the fused layer has the same scheme.
109 - if proj_name in _FUSED_LAYER_NAME_MAPPING:
110 - shard_proj_names = _FUSED_LAYER_NAME_MAPPING[proj_name]
105 + if proj_name in FUSED_LAYER_NAME_MAPPING:
106 + shard_proj_names = FUSED_LAYER_NAME_MAPPING[proj_name]
107
108 # Convert fused_name --> [shard_names]
109 shard_names = [
vllm/model_executor/layers/quantization/fbgemm_fp8.py
+3 -36
@@ -11,6 +11,8 @@ from vllm.model_executor.layers.quantization.base_config import (
11 QuantizationConfig, QuantizeMethodBase)
12 from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
13 apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
14 +from vllm.model_executor.layers.quantization.utils.quant_utils import (
15 + is_layer_skipped)
16 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
17 apply_fp8_linear, create_per_channel_scale_param)
18 from vllm.model_executor.utils import set_weight_attrs
@@ -18,14 +20,6 @@ from vllm.platforms import current_platform
20
21 logger = init_logger(__name__)
22
21 -# Note: this is a hack. We should update each model to register the
22 -# stacked params and get it from there instead in a future PR.
23 -# fused_name: List[shard_name]
24 -_FUSED_LAYER_NAME_MAPPING = {
25 - "qkv_proj": ["q_proj", "k_proj", "v_proj"],
26 - "gate_up_proj": ["gate_proj", "up_proj"]
27 -}
28 -
23
24 class FBGEMMFp8Config(QuantizationConfig):
25 """Config class for FBGEMM Fp8."""
@@ -62,37 +56,10 @@ class FBGEMMFp8Config(QuantizationConfig):
56 input_scale_ub = cls.get_from_keys(config, ["activation_scale_ub"])
57 return cls(ignore_list=ignore_list, input_scale_ub=input_scale_ub)
58
65 - def _is_layer_skipped(self, prefix: str) -> bool:
66 - # prefix: model.layers.0.self_attn.q_proj
67 - # proj_name: q_proj
68 - proj_name = prefix.split(".")[-1]
69 - if proj_name in _FUSED_LAYER_NAME_MAPPING:
70 - shard_prefixes = [
71 - prefix.replace(proj_name, shard_proj_name)
72 - for shard_proj_name in _FUSED_LAYER_NAME_MAPPING[proj_name]
73 - ]
74 -
75 - is_skipped = None
76 - for shard_prefix in shard_prefixes:
77 - is_shard_skipped = shard_prefix in self.ignore_list
78 -
79 - if is_skipped is None:
80 - is_skipped = is_shard_skipped
81 - elif is_shard_skipped != is_skipped:
82 - raise ValueError(
83 - f"Detected some but not all shards of {prefix} "
84 - "are quantized. All shards of fused layers "
85 - "to have the same precision.")
86 - else:
87 - is_skipped = prefix in self.ignore_list
88 -
89 - assert is_skipped is not None
90 - return is_skipped
91 -
59 def get_quant_method(self, layer: torch.nn.Module,
60 prefix: str) -> Optional["QuantizeMethodBase"]:
61 if isinstance(layer, LinearBase):
95 - if self._is_layer_skipped(prefix):
62 + if is_layer_skipped(prefix, self.ignore_list):
63 return UnquantizedLinearMethod()
64 return FBGEMMFp8LinearMethod(self)
65 return None
vllm/model_executor/layers/quantization/fp8.py
+11 -2
@@ -8,12 +8,15 @@ from vllm import _custom_ops as ops
8 from vllm.logger import init_logger
9 from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
10 fused_moe)
11 -from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
11 +from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
12 + UnquantizedLinearMethod)
13 from vllm.model_executor.layers.quantization.base_config import (
14 QuantizationConfig, QuantizeMethodBase)
15 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
16 from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
17 apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
18 +from vllm.model_executor.layers.quantization.utils.quant_utils import (
19 + is_layer_skipped)
20 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
21 all_close_1d, apply_fp8_linear, create_per_tensor_scale_param,
22 cutlass_fp8_supported, per_tensor_dequantize, requantize_with_max_scale)
@@ -33,6 +36,7 @@ class Fp8Config(QuantizationConfig):
36 self,
37 is_checkpoint_fp8_serialized: bool = False,
38 activation_scheme: str = "dynamic",
39 + ignored_layers: Optional[List[str]] = None,
40 ) -> None:
41 self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized
42 if is_checkpoint_fp8_serialized:
@@ -42,6 +46,7 @@ class Fp8Config(QuantizationConfig):
46 raise ValueError(
47 f"Unsupported activation scheme {activation_scheme}")
48 self.activation_scheme = activation_scheme
49 + self.ignored_layers = ignored_layers or []
50
51 @classmethod
52 def get_name(cls) -> str:
@@ -64,14 +69,18 @@ class Fp8Config(QuantizationConfig):
69 quant_method = cls.get_from_keys(config, ["quant_method"])
70 is_checkpoint_fp8_serialized = ("fp8" in quant_method)
71 activation_scheme = cls.get_from_keys(config, ["activation_scheme"])
72 + ignored_layers = cls.get_from_keys_or(config, ["ignored_layers"], None)
73 return cls(is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized,
68 - activation_scheme=activation_scheme)
74 + activation_scheme=activation_scheme,
75 + ignored_layers=ignored_layers)
76
77 def get_quant_method(self, layer: torch.nn.Module,
78 prefix: str) -> Optional["QuantizeMethodBase"]:
79 from vllm.attention.layer import Attention # Avoid circular import
80
81 if isinstance(layer, LinearBase):
82 + if is_layer_skipped(prefix, self.ignored_layers):
83 + return UnquantizedLinearMethod()
84 return Fp8LinearMethod(self)
85 elif isinstance(layer, FusedMoE):
86 return Fp8MoEMethod(self)
vllm/model_executor/layers/quantization/utils/quant_utils.py
+38
@@ -1,10 +1,48 @@
1 """This file is used for /tests and /benchmarks"""
2 +from typing import List
3 +
4 import numpy
5 import torch
6
7 SUPPORTED_NUM_BITS = [4, 8]
8 SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
9
10 +# Note: this is a hack. We should update each model to register the
11 +# stacked params and get it from there instead in a future PR.
12 +# fused_name: List[shard_name]
13 +FUSED_LAYER_NAME_MAPPING = {
14 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],
15 + "gate_up_proj": ["gate_proj", "up_proj"]
16 +}
17 +
18 +
19 +def is_layer_skipped(prefix: str, ignored_layers: List[str]) -> bool:
20 + # prefix: model.layers.0.self_attn.q_proj
21 + # proj_name: q_proj
22 + proj_name = prefix.split(".")[-1]
23 + if proj_name in FUSED_LAYER_NAME_MAPPING:
24 + shard_prefixes = [
25 + prefix.replace(proj_name, shard_proj_name)
26 + for shard_proj_name in FUSED_LAYER_NAME_MAPPING[proj_name]
27 + ]
28 +
29 + is_skipped = None
30 + for shard_prefix in shard_prefixes:
31 + is_shard_skipped = shard_prefix in ignored_layers
32 +
33 + if is_skipped is None:
34 + is_skipped = is_shard_skipped
35 + elif is_shard_skipped != is_skipped:
36 + raise ValueError(
37 + f"Detected some but not all shards of {prefix} "
38 + "are quantized. All shards of fused layers "
39 + "to have the same precision.")
40 + else:
41 + is_skipped = prefix in ignored_layers
42 +
43 + assert is_skipped is not None
44 + return is_skipped
45 +
46
47 def get_pack_factor(num_bits):
48 assert num_bits in SUPPORTED_NUM_BITS, f"Unsupported num_bits = {num_bits}"