A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Quant][Perf] Use moe_wna16 kernel by default for MoEs with many experts (#13236)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Feb 14, 2025 at 15:53 UTC
5e5c8e091eacc16672a0a8265eb5cb0ece85d24b
4 files changed
+39
-26
tests/weight_loading/test_weight_loading.py
+1
-1
@@ -12,7 +12,7 @@ MODEL_NAME = os.environ.get("MODEL_NAME",
12
"robertgshaw2/zephyr-7b-beta-channelwise-gptq")
13
REVISION = os.environ.get("REVISION", "main")
14
QUANTIZATION = os.environ.get("QUANTIZATION", "gptq_marlin")
15
-MIN_CAPABILITY = os.environ.get("MIN_CAPABILITY", "89")
15
+MIN_CAPABILITY = os.environ.get("MIN_CAPABILITY", "80")
16
17
18
@pytest.mark.skipif(
vllm/model_executor/layers/quantization/awq_marlin.py
+7
-1
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.quantization.awq import (AWQConfig,
17
is_layer_skipped_awq)
18
from vllm.model_executor.layers.quantization.base_config import (
19
QuantizationConfig, QuantizeMethodBase)
20
+from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
21
from vllm.model_executor.layers.quantization.utils import replace_parameter
22
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
23
apply_awq_marlin_linear, awq_to_marlin_zero_points, check_marlin_supported,
@@ -134,7 +135,12 @@ class AWQMarlinConfig(QuantizationConfig):
135
self.full_config).get_quant_method(layer, prefix)
136
return AWQMarlinLinearMethod(self)
137
elif isinstance(layer, FusedMoE):
137
- return AWQMoEMethod(self)
138
+ if layer.num_experts > 32:
139
+ # For MoEs with many experts the moe_wna16 kernel is faster
140
+ return MoeWNA16Config.from_config(
141
+ self.full_config).get_quant_method(layer, prefix)
142
+ else:
143
+ return AWQMoEMethod(self)
144
return None
145
146
@classmethod
vllm/model_executor/layers/quantization/gptq_marlin.py
+16
-19
@@ -10,20 +10,18 @@ from vllm.logger import init_logger
10
from vllm.model_executor.layers.fused_moe.layer import (
11
FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
12
from vllm.model_executor.layers.linear import (LinearMethodBase,
13
- UnquantizedLinearMethod,
13
set_weight_attrs)
14
from vllm.model_executor.layers.quantization.base_config import (
16
- QuantizationConfig)
15
+ QuantizationConfig, QuantizeMethodBase)
16
from vllm.model_executor.layers.quantization.kernels.mixed_precision import (
17
MPLinearLayerConfig, choose_mp_linear_kernel)
18
+from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
19
from vllm.model_executor.layers.quantization.utils import replace_parameter
20
from vllm.model_executor.layers.quantization.utils.gptq_utils import (
21
get_linear_quant_method)
22
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
23
check_marlin_supported, marlin_moe_permute_scales,
24
marlin_repeat_scales_on_all_ranks, verify_marlin_supported)
25
-from vllm.model_executor.layers.vocab_parallel_embedding import (
26
- UnquantizedEmbeddingMethod)
25
from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
26
GroupQuantScaleParameter,
27
PackedColumnParameter,
@@ -44,15 +42,10 @@ class GPTQMarlinConfig(QuantizationConfig):
42
(8, True): scalar_types.uint8b128,
43
}
44
47
- def __init__(
48
- self,
49
- weight_bits: int,
50
- group_size: int,
51
- desc_act: bool,
52
- is_sym: bool,
53
- lm_head_quantized: bool,
54
- dynamic: Dict[str, Dict[str, Union[int, bool]]],
55
- ) -> None:
45
+ def __init__(self, weight_bits: int, group_size: int, desc_act: bool,
46
+ is_sym: bool, lm_head_quantized: bool,
47
+ dynamic: Dict[str, Dict[str, Union[int, bool]]],
48
+ full_config: Dict[str, Any]) -> None:
49
if desc_act and group_size == -1:
50
# In this case, act_order == True is the same as act_order == False
51
# (since we have only one group per output channel)
@@ -90,6 +83,7 @@ class GPTQMarlinConfig(QuantizationConfig):
83
self.group_size = group_size
84
self.desc_act = desc_act
85
self.lm_head_quantized = lm_head_quantized
86
+ self.full_config = full_config
87
88
if (weight_bits, is_sym) not in self.TYPE_MAP:
89
raise ValueError("Unsupported quantization config: "
@@ -132,7 +126,7 @@ class GPTQMarlinConfig(QuantizationConfig):
126
lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
127
default=False)
128
return cls(weight_bits, group_size, desc_act, is_sym,
135
- lm_head_quantized, dynamic)
129
+ lm_head_quantized, dynamic, config)
130
131
@classmethod
132
def override_quantization_method(cls, hf_quant_cfg,
@@ -155,12 +149,15 @@ class GPTQMarlinConfig(QuantizationConfig):
149
" faster inference")
150
return None
151
158
- def get_quant_method(
159
- self, layer: torch.nn.Module, prefix: str
160
- ) -> Optional[Union["GPTQMarlinLinearMethod", "GPTQMarlinMoEMethod",
161
- UnquantizedLinearMethod, UnquantizedEmbeddingMethod]]:
152
+ def get_quant_method(self, layer: torch.nn.Module,
153
+ prefix: str) -> Optional["QuantizeMethodBase"]:
154
if isinstance(layer, FusedMoE):
163
- return GPTQMarlinMoEMethod(self)
155
+ if layer.num_experts > 32:
156
+ # For MoEs with many experts the moe_wna16 kernel is faster
157
+ return MoeWNA16Config.from_config(
158
+ self.full_config).get_quant_method(layer, prefix)
159
+ else:
160
+ return GPTQMarlinMoEMethod(self)
161
return get_linear_quant_method(self, layer, prefix,
162
GPTQMarlinLinearMethod)
163
vllm/model_executor/layers/quantization/moe_wna16.py
+15
-5
@@ -9,13 +9,8 @@ from vllm.model_executor.layers.fused_moe.layer import (
9
FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
10
from vllm.model_executor.layers.linear import (LinearBase,
11
UnquantizedLinearMethod)
12
-from vllm.model_executor.layers.quantization.awq import AWQConfig
13
-from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig, QuantizeMethodBase)
16
-from vllm.model_executor.layers.quantization.gptq import GPTQConfig
17
-from vllm.model_executor.layers.quantization.gptq_marlin import (
18
- GPTQMarlinConfig)
14
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
15
check_marlin_supports_layer)
16
from vllm.model_executor.utils import set_weight_attrs
@@ -37,6 +32,12 @@ class MoeWNA16Config(QuantizationConfig):
32
self.linear_quant_method = linear_quant_method
33
self.full_config = full_config
34
self.use_marlin = False
35
+ # Avoid circular import
36
+ from vllm.model_executor.layers.quantization.awq import AWQConfig
37
+ from vllm.model_executor.layers.quantization.awq_marlin import (
38
+ AWQMarlinConfig)
39
+ from vllm.model_executor.layers.quantization.gptq_marlin import (
40
+ GPTQMarlinConfig)
41
if self.linear_quant_method == "gptq":
42
self.use_marlin = GPTQMarlinConfig.is_gptq_marlin_compatible(
43
full_config)
@@ -115,6 +116,8 @@ class MoeWNA16Config(QuantizationConfig):
116
capability_tuple = current_platform.get_device_capability()
117
device_capability = (-1 if capability_tuple is None else
118
capability_tuple.to_int())
119
+ # Avoid circular import
120
+ from vllm.model_executor.layers.quantization.awq import AWQConfig
121
awq_min_capability = AWQConfig.get_min_capability()
122
123
gptq_compatible = quant_method == "gptq" and \
@@ -129,6 +132,13 @@ class MoeWNA16Config(QuantizationConfig):
132
if is_layer_skipped_quant(prefix, self.modules_to_not_convert):
133
return UnquantizedLinearMethod()
134
elif isinstance(layer, LinearBase):
135
+ # Avoid circular import
136
+ from vllm.model_executor.layers.quantization.awq import AWQConfig
137
+ from vllm.model_executor.layers.quantization.awq_marlin import (
138
+ AWQMarlinConfig)
139
+ from vllm.model_executor.layers.quantization.gptq import GPTQConfig
140
+ from vllm.model_executor.layers.quantization.gptq_marlin import (
141
+ GPTQMarlinConfig)
142
if self.linear_quant_method == "gptq":
143
if self.use_marlin:
144
return GPTQMarlinConfig.from_config(