@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5e5c8e091e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Quant][Perf] Use moe_wna16 kernel by default for MoEs with many experts (#13236)

Signed-off-by: mgoin <mgoin64@gmail.com>

Michael Goin committed Feb 14, 2025 at 15:53 UTC 5e5c8e091eacc16672a0a8265eb5cb0ece85d24b
4 files changed +39 -26
tests/weight_loading/test_weight_loading.py
+1 -1
@@ -12,7 +12,7 @@ MODEL_NAME = os.environ.get("MODEL_NAME",
12 "robertgshaw2/zephyr-7b-beta-channelwise-gptq")
13 REVISION = os.environ.get("REVISION", "main")
14 QUANTIZATION = os.environ.get("QUANTIZATION", "gptq_marlin")
15 -MIN_CAPABILITY = os.environ.get("MIN_CAPABILITY", "89")
15 +MIN_CAPABILITY = os.environ.get("MIN_CAPABILITY", "80")
16
17
18 @pytest.mark.skipif(
vllm/model_executor/layers/quantization/awq_marlin.py
+7 -1
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.quantization.awq import (AWQConfig,
17 is_layer_skipped_awq)
18 from vllm.model_executor.layers.quantization.base_config import (
19 QuantizationConfig, QuantizeMethodBase)
20 +from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
21 from vllm.model_executor.layers.quantization.utils import replace_parameter
22 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
23 apply_awq_marlin_linear, awq_to_marlin_zero_points, check_marlin_supported,
@@ -134,7 +135,12 @@ class AWQMarlinConfig(QuantizationConfig):
135 self.full_config).get_quant_method(layer, prefix)
136 return AWQMarlinLinearMethod(self)
137 elif isinstance(layer, FusedMoE):
137 - return AWQMoEMethod(self)
138 + if layer.num_experts > 32:
139 + # For MoEs with many experts the moe_wna16 kernel is faster
140 + return MoeWNA16Config.from_config(
141 + self.full_config).get_quant_method(layer, prefix)
142 + else:
143 + return AWQMoEMethod(self)
144 return None
145
146 @classmethod
vllm/model_executor/layers/quantization/gptq_marlin.py
+16 -19
@@ -10,20 +10,18 @@ from vllm.logger import init_logger
10 from vllm.model_executor.layers.fused_moe.layer import (
11 FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
12 from vllm.model_executor.layers.linear import (LinearMethodBase,
13 - UnquantizedLinearMethod,
13 set_weight_attrs)
14 from vllm.model_executor.layers.quantization.base_config import (
16 - QuantizationConfig)
15 + QuantizationConfig, QuantizeMethodBase)
16 from vllm.model_executor.layers.quantization.kernels.mixed_precision import (
17 MPLinearLayerConfig, choose_mp_linear_kernel)
18 +from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
19 from vllm.model_executor.layers.quantization.utils import replace_parameter
20 from vllm.model_executor.layers.quantization.utils.gptq_utils import (
21 get_linear_quant_method)
22 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
23 check_marlin_supported, marlin_moe_permute_scales,
24 marlin_repeat_scales_on_all_ranks, verify_marlin_supported)
25 -from vllm.model_executor.layers.vocab_parallel_embedding import (
26 - UnquantizedEmbeddingMethod)
25 from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
26 GroupQuantScaleParameter,
27 PackedColumnParameter,
@@ -44,15 +42,10 @@ class GPTQMarlinConfig(QuantizationConfig):
42 (8, True): scalar_types.uint8b128,
43 }
44
47 - def __init__(
48 - self,
49 - weight_bits: int,
50 - group_size: int,
51 - desc_act: bool,
52 - is_sym: bool,
53 - lm_head_quantized: bool,
54 - dynamic: Dict[str, Dict[str, Union[int, bool]]],
55 - ) -> None:
45 + def __init__(self, weight_bits: int, group_size: int, desc_act: bool,
46 + is_sym: bool, lm_head_quantized: bool,
47 + dynamic: Dict[str, Dict[str, Union[int, bool]]],
48 + full_config: Dict[str, Any]) -> None:
49 if desc_act and group_size == -1:
50 # In this case, act_order == True is the same as act_order == False
51 # (since we have only one group per output channel)
@@ -90,6 +83,7 @@ class GPTQMarlinConfig(QuantizationConfig):
83 self.group_size = group_size
84 self.desc_act = desc_act
85 self.lm_head_quantized = lm_head_quantized
86 + self.full_config = full_config
87
88 if (weight_bits, is_sym) not in self.TYPE_MAP:
89 raise ValueError("Unsupported quantization config: "
@@ -132,7 +126,7 @@ class GPTQMarlinConfig(QuantizationConfig):
126 lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
127 default=False)
128 return cls(weight_bits, group_size, desc_act, is_sym,
135 - lm_head_quantized, dynamic)
129 + lm_head_quantized, dynamic, config)
130
131 @classmethod
132 def override_quantization_method(cls, hf_quant_cfg,
@@ -155,12 +149,15 @@ class GPTQMarlinConfig(QuantizationConfig):
149 " faster inference")
150 return None
151
158 - def get_quant_method(
159 - self, layer: torch.nn.Module, prefix: str
160 - ) -> Optional[Union["GPTQMarlinLinearMethod", "GPTQMarlinMoEMethod",
161 - UnquantizedLinearMethod, UnquantizedEmbeddingMethod]]:
152 + def get_quant_method(self, layer: torch.nn.Module,
153 + prefix: str) -> Optional["QuantizeMethodBase"]:
154 if isinstance(layer, FusedMoE):
163 - return GPTQMarlinMoEMethod(self)
155 + if layer.num_experts > 32:
156 + # For MoEs with many experts the moe_wna16 kernel is faster
157 + return MoeWNA16Config.from_config(
158 + self.full_config).get_quant_method(layer, prefix)
159 + else:
160 + return GPTQMarlinMoEMethod(self)
161 return get_linear_quant_method(self, layer, prefix,
162 GPTQMarlinLinearMethod)
163
vllm/model_executor/layers/quantization/moe_wna16.py
+15 -5
@@ -9,13 +9,8 @@ from vllm.model_executor.layers.fused_moe.layer import (
9 FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
10 from vllm.model_executor.layers.linear import (LinearBase,
11 UnquantizedLinearMethod)
12 -from vllm.model_executor.layers.quantization.awq import AWQConfig
13 -from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig, QuantizeMethodBase)
16 -from vllm.model_executor.layers.quantization.gptq import GPTQConfig
17 -from vllm.model_executor.layers.quantization.gptq_marlin import (
18 - GPTQMarlinConfig)
14 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
15 check_marlin_supports_layer)
16 from vllm.model_executor.utils import set_weight_attrs
@@ -37,6 +32,12 @@ class MoeWNA16Config(QuantizationConfig):
32 self.linear_quant_method = linear_quant_method
33 self.full_config = full_config
34 self.use_marlin = False
35 + # Avoid circular import
36 + from vllm.model_executor.layers.quantization.awq import AWQConfig
37 + from vllm.model_executor.layers.quantization.awq_marlin import (
38 + AWQMarlinConfig)
39 + from vllm.model_executor.layers.quantization.gptq_marlin import (
40 + GPTQMarlinConfig)
41 if self.linear_quant_method == "gptq":
42 self.use_marlin = GPTQMarlinConfig.is_gptq_marlin_compatible(
43 full_config)
@@ -115,6 +116,8 @@ class MoeWNA16Config(QuantizationConfig):
116 capability_tuple = current_platform.get_device_capability()
117 device_capability = (-1 if capability_tuple is None else
118 capability_tuple.to_int())
119 + # Avoid circular import
120 + from vllm.model_executor.layers.quantization.awq import AWQConfig
121 awq_min_capability = AWQConfig.get_min_capability()
122
123 gptq_compatible = quant_method == "gptq" and \
@@ -129,6 +132,13 @@ class MoeWNA16Config(QuantizationConfig):
132 if is_layer_skipped_quant(prefix, self.modules_to_not_convert):
133 return UnquantizedLinearMethod()
134 elif isinstance(layer, LinearBase):
135 + # Avoid circular import
136 + from vllm.model_executor.layers.quantization.awq import AWQConfig
137 + from vllm.model_executor.layers.quantization.awq_marlin import (
138 + AWQMarlinConfig)
139 + from vllm.model_executor.layers.quantization.gptq import GPTQConfig
140 + from vllm.model_executor.layers.quantization.gptq_marlin import (
141 + GPTQMarlinConfig)
142 if self.linear_quant_method == "gptq":
143 if self.use_marlin:
144 return GPTQMarlinConfig.from_config(