@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 09972e716c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Bugfix] Allow fallback to AWQ from AWQMarlin at per-layer granularity (#13119)

Michael Goin committed Feb 12, 2025 at 12:19 UTC 09972e716c4a90bfd4385540c9f478e18b4efb2d
4 files changed +58 -29
vllm/model_executor/layers/linear.py
+19 -16
@@ -290,29 +290,30 @@ class ColumnParallelLinear(LinearBase):
290 quant_config: Optional[QuantizationConfig] = None,
291 output_sizes: Optional[list[int]] = None,
292 prefix: str = ""):
293 - super().__init__(input_size, output_size, skip_bias_add, params_dtype,
294 - quant_config, prefix)
295 -
296 - self.gather_output = gather_output
297 -
293 # Divide the weight matrix along the last dimension.
299 - tp_size = get_tensor_model_parallel_world_size()
300 - assert self.quant_method is not None
301 - self.output_size_per_partition = divide(self.output_size, tp_size)
294 + self.tp_size = get_tensor_model_parallel_world_size()
295 + self.input_size_per_partition = input_size
296 + self.output_size_per_partition = divide(output_size, self.tp_size)
297 self.output_partition_sizes = [self.output_size_per_partition]
298 # If QKV or MergedColumn, use output size of each partition.
299 if hasattr(self, "output_sizes"):
300 self.output_partition_sizes = [
306 - divide(output_size, tp_size)
301 + divide(output_size, self.tp_size)
302 for output_size in self.output_sizes
303 ]
304
305 + super().__init__(input_size, output_size, skip_bias_add, params_dtype,
306 + quant_config, prefix)
307 +
308 + self.gather_output = gather_output
309 +
310 if output_sizes is None:
311 output_sizes = [output_size]
312
313 + assert self.quant_method is not None
314 self.quant_method.create_weights(
315 layer=self,
315 - input_size_per_partition=self.input_size,
316 + input_size_per_partition=self.input_size_per_partition,
317 output_partition_sizes=self.output_partition_sizes,
318 input_size=self.input_size,
319 output_size=self.output_size,
@@ -1044,22 +1045,24 @@ class RowParallelLinear(LinearBase):
1045 reduce_results: bool = True,
1046 quant_config: Optional[QuantizationConfig] = None,
1047 prefix: str = ""):
1048 + # Divide the weight matrix along the first dimension.
1049 + self.tp_rank = get_tensor_model_parallel_rank()
1050 + self.tp_size = get_tensor_model_parallel_world_size()
1051 + self.input_size_per_partition = divide(input_size, self.tp_size)
1052 + self.output_size_per_partition = output_size
1053 + self.output_partition_sizes = [output_size]
1054 +
1055 super().__init__(input_size, output_size, skip_bias_add, params_dtype,
1056 quant_config, prefix)
1057
1058 self.input_is_parallel = input_is_parallel
1059 self.reduce_results = reduce_results
1060
1053 - # Divide the weight matrix along the last dimension.
1054 - self.tp_rank = get_tensor_model_parallel_rank()
1055 - self.tp_size = get_tensor_model_parallel_world_size()
1056 - self.input_size_per_partition = divide(input_size, self.tp_size)
1061 assert self.quant_method is not None
1058 -
1062 self.quant_method.create_weights(
1063 layer=self,
1064 input_size_per_partition=self.input_size_per_partition,
1062 - output_partition_sizes=[self.output_size],
1065 + output_partition_sizes=self.output_partition_sizes,
1066 input_size=self.input_size,
1067 output_size=self.output_size,
1068 params_dtype=self.params_dtype,
vllm/model_executor/layers/quantization/awq_marlin.py
+18 -10
@@ -13,15 +13,17 @@ from vllm.model_executor.layers.fused_moe.layer import (
13 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
14 UnquantizedLinearMethod,
15 set_weight_attrs)
16 -from vllm.model_executor.layers.quantization.awq import is_layer_skipped_awq
16 +from vllm.model_executor.layers.quantization.awq import (AWQConfig,
17 + is_layer_skipped_awq)
18 from vllm.model_executor.layers.quantization.base_config import (
19 QuantizationConfig, QuantizeMethodBase)
20 from vllm.model_executor.layers.quantization.utils import replace_parameter
21 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
22 apply_awq_marlin_linear, awq_to_marlin_zero_points, check_marlin_supported,
22 - marlin_make_empty_g_idx, marlin_make_workspace, marlin_moe_permute_scales,
23 - marlin_permute_scales, moe_awq_to_marlin_zero_points,
24 - verify_marlin_supported, verify_marlin_supports_shape)
23 + check_marlin_supports_layer, marlin_make_empty_g_idx,
24 + marlin_make_workspace, marlin_moe_permute_scales, marlin_permute_scales,
25 + moe_awq_to_marlin_zero_points, verify_marlin_supported,
26 + verify_marlin_supports_shape)
27 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
28 from vllm.model_executor.parameter import (GroupQuantScaleParameter,
29 PackedvLLMParameter)
@@ -40,18 +42,17 @@ class AWQMarlinConfig(QuantizationConfig):
42 8: scalar_types.uint8,
43 }
44
43 - def __init__(self,
44 - weight_bits: int,
45 - group_size: int,
46 - zero_point: bool,
45 + def __init__(self, weight_bits: int, group_size: int, zero_point: bool,
46 lm_head_quantized: bool,
48 - modules_to_not_convert: Optional[List[str]] = None) -> None:
47 + modules_to_not_convert: Optional[List[str]],
48 + full_config: Dict[str, Any]) -> None:
49 self.pack_factor = 32 // weight_bits # packed into int32
50 self.group_size = group_size
51 self.zero_point = zero_point
52 self.lm_head_quantized = lm_head_quantized
53 self.weight_bits = weight_bits
54 self.modules_to_not_convert = modules_to_not_convert or []
55 + self.full_config = full_config
56
57 if self.weight_bits not in self.TYPE_MAP:
58 raise ValueError(f"Unsupported num_bits = {self.weight_bits}. "
@@ -96,7 +97,7 @@ class AWQMarlinConfig(QuantizationConfig):
97 modules_to_not_convert = cls.get_from_keys_or(
98 config, ["modules_to_not_convert"], None)
99 return cls(weight_bits, group_size, zero_point, lm_head_quantized,
99 - modules_to_not_convert)
100 + modules_to_not_convert, config)
101
102 @classmethod
103 def override_quantization_method(cls, hf_quant_cfg,
@@ -124,6 +125,13 @@ class AWQMarlinConfig(QuantizationConfig):
125 (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
126 if is_layer_skipped_awq(prefix, self.modules_to_not_convert):
127 return UnquantizedLinearMethod()
128 + # Check if the layer is supported by AWQMarlin.
129 + if not check_marlin_supports_layer(layer, self.group_size):
130 + logger.warning_once(
131 + f"Layer '{prefix}' is not supported by AWQMarlin. "
132 + "Falling back to unoptimized AWQ kernels.")
133 + return AWQConfig.from_config(
134 + self.full_config).get_quant_method(layer, prefix)
135 return AWQMarlinLinearMethod(self)
136 elif isinstance(layer, FusedMoE):
137 return AWQMoEMethod(self)
vllm/model_executor/layers/quantization/moe_wna16.py
+6 -3
@@ -16,6 +16,8 @@ from vllm.model_executor.layers.quantization.base_config import (
16 from vllm.model_executor.layers.quantization.gptq import GPTQConfig
17 from vllm.model_executor.layers.quantization.gptq_marlin import (
18 GPTQMarlinConfig)
19 +from vllm.model_executor.layers.quantization.utils.marlin_utils import (
20 + check_marlin_supports_layer)
21 from vllm.model_executor.utils import set_weight_attrs
22 from vllm.platforms import current_platform
23
@@ -87,8 +89,8 @@ class MoeWNA16Config(QuantizationConfig):
89 modules_to_not_convert = []
90 elif linear_quant_method == "awq":
91 has_zp = cls.get_from_keys(config, ["zero_point"])
90 - modules_to_not_convert = cls.get_from_keys(
91 - config, ["modules_to_not_convert"])
92 + modules_to_not_convert = cls.get_from_keys_or(
93 + config, ["modules_to_not_convert"], None)
94 else:
95 raise ValueError("moe_wna16 only support gptq and awq.")
96
@@ -135,7 +137,8 @@ class MoeWNA16Config(QuantizationConfig):
137 return GPTQConfig.from_config(
138 self.full_config).get_quant_method(layer, prefix)
139 elif self.linear_quant_method == "awq":
138 - if self.use_marlin:
140 + if self.use_marlin and check_marlin_supports_layer(
141 + layer, self.group_size):
142 return AWQMarlinConfig.from_config(
143 self.full_config).get_quant_method(layer, prefix)
144 else:
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+15
@@ -6,6 +6,7 @@ import numpy
6 import torch
7
8 from vllm import _custom_ops as ops
9 +from vllm.model_executor.layers.linear import LinearBase
10 from vllm.platforms import current_platform
11 from vllm.scalar_type import ScalarType, scalar_types
12
@@ -135,6 +136,20 @@ def check_marlin_supports_shape(output_size_per_partition: int,
136 return True, None
137
138
139 +def check_marlin_supports_layer(layer: LinearBase, group_size: int) \
140 + -> bool:
141 + output_size_per_partition = getattr(layer, "output_size_per_partition",
142 + None) or layer.output_size
143 + input_size_per_partition = getattr(layer, "input_size_per_partition",
144 + None) or layer.input_size
145 +
146 + return check_marlin_supports_shape(
147 + output_size_per_partition=output_size_per_partition,
148 + input_size_per_partition=input_size_per_partition,
149 + input_size=layer.input_size,
150 + group_size=group_size)[0]
151 +
152 +
153 def marlin_make_workspace(output_size_per_partition: int,
154 device: torch.device) -> torch.Tensor:
155 max_workspace_size = (output_size_per_partition //