A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Bugfix] Allow fallback to AWQ from AWQMarlin at per-layer granularity (#13119)
Michael Goin committed
Feb 12, 2025 at 12:19 UTC
09972e716c4a90bfd4385540c9f478e18b4efb2d
4 files changed
+58
-29
vllm/model_executor/layers/linear.py
+19
-16
@@ -290,29 +290,30 @@ class ColumnParallelLinear(LinearBase):
290
quant_config: Optional[QuantizationConfig] = None,
291
output_sizes: Optional[list[int]] = None,
292
prefix: str = ""):
293
- super().__init__(input_size, output_size, skip_bias_add, params_dtype,
294
- quant_config, prefix)
295
-
296
- self.gather_output = gather_output
297
-
293
# Divide the weight matrix along the last dimension.
299
- tp_size = get_tensor_model_parallel_world_size()
300
- assert self.quant_method is not None
301
- self.output_size_per_partition = divide(self.output_size, tp_size)
294
+ self.tp_size = get_tensor_model_parallel_world_size()
295
+ self.input_size_per_partition = input_size
296
+ self.output_size_per_partition = divide(output_size, self.tp_size)
297
self.output_partition_sizes = [self.output_size_per_partition]
298
# If QKV or MergedColumn, use output size of each partition.
299
if hasattr(self, "output_sizes"):
300
self.output_partition_sizes = [
306
- divide(output_size, tp_size)
301
+ divide(output_size, self.tp_size)
302
for output_size in self.output_sizes
303
]
304
305
+ super().__init__(input_size, output_size, skip_bias_add, params_dtype,
306
+ quant_config, prefix)
307
+
308
+ self.gather_output = gather_output
309
+
310
if output_sizes is None:
311
output_sizes = [output_size]
312
313
+ assert self.quant_method is not None
314
self.quant_method.create_weights(
315
layer=self,
315
- input_size_per_partition=self.input_size,
316
+ input_size_per_partition=self.input_size_per_partition,
317
output_partition_sizes=self.output_partition_sizes,
318
input_size=self.input_size,
319
output_size=self.output_size,
@@ -1044,22 +1045,24 @@ class RowParallelLinear(LinearBase):
1045
reduce_results: bool = True,
1046
quant_config: Optional[QuantizationConfig] = None,
1047
prefix: str = ""):
1048
+ # Divide the weight matrix along the first dimension.
1049
+ self.tp_rank = get_tensor_model_parallel_rank()
1050
+ self.tp_size = get_tensor_model_parallel_world_size()
1051
+ self.input_size_per_partition = divide(input_size, self.tp_size)
1052
+ self.output_size_per_partition = output_size
1053
+ self.output_partition_sizes = [output_size]
1054
+
1055
super().__init__(input_size, output_size, skip_bias_add, params_dtype,
1056
quant_config, prefix)
1057
1058
self.input_is_parallel = input_is_parallel
1059
self.reduce_results = reduce_results
1060
1053
- # Divide the weight matrix along the last dimension.
1054
- self.tp_rank = get_tensor_model_parallel_rank()
1055
- self.tp_size = get_tensor_model_parallel_world_size()
1056
- self.input_size_per_partition = divide(input_size, self.tp_size)
1061
assert self.quant_method is not None
1058
-
1062
self.quant_method.create_weights(
1063
layer=self,
1064
input_size_per_partition=self.input_size_per_partition,
1062
- output_partition_sizes=[self.output_size],
1065
+ output_partition_sizes=self.output_partition_sizes,
1066
input_size=self.input_size,
1067
output_size=self.output_size,
1068
params_dtype=self.params_dtype,
vllm/model_executor/layers/quantization/awq_marlin.py
+18
-10
@@ -13,15 +13,17 @@ from vllm.model_executor.layers.fused_moe.layer import (
13
from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
14
UnquantizedLinearMethod,
15
set_weight_attrs)
16
-from vllm.model_executor.layers.quantization.awq import is_layer_skipped_awq
16
+from vllm.model_executor.layers.quantization.awq import (AWQConfig,
17
+ is_layer_skipped_awq)
18
from vllm.model_executor.layers.quantization.base_config import (
19
QuantizationConfig, QuantizeMethodBase)
20
from vllm.model_executor.layers.quantization.utils import replace_parameter
21
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
22
apply_awq_marlin_linear, awq_to_marlin_zero_points, check_marlin_supported,
22
- marlin_make_empty_g_idx, marlin_make_workspace, marlin_moe_permute_scales,
23
- marlin_permute_scales, moe_awq_to_marlin_zero_points,
24
- verify_marlin_supported, verify_marlin_supports_shape)
23
+ check_marlin_supports_layer, marlin_make_empty_g_idx,
24
+ marlin_make_workspace, marlin_moe_permute_scales, marlin_permute_scales,
25
+ moe_awq_to_marlin_zero_points, verify_marlin_supported,
26
+ verify_marlin_supports_shape)
27
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
28
from vllm.model_executor.parameter import (GroupQuantScaleParameter,
29
PackedvLLMParameter)
@@ -40,18 +42,17 @@ class AWQMarlinConfig(QuantizationConfig):
42
8: scalar_types.uint8,
43
}
44
43
- def __init__(self,
44
- weight_bits: int,
45
- group_size: int,
46
- zero_point: bool,
45
+ def __init__(self, weight_bits: int, group_size: int, zero_point: bool,
46
lm_head_quantized: bool,
48
- modules_to_not_convert: Optional[List[str]] = None) -> None:
47
+ modules_to_not_convert: Optional[List[str]],
48
+ full_config: Dict[str, Any]) -> None:
49
self.pack_factor = 32 // weight_bits # packed into int32
50
self.group_size = group_size
51
self.zero_point = zero_point
52
self.lm_head_quantized = lm_head_quantized
53
self.weight_bits = weight_bits
54
self.modules_to_not_convert = modules_to_not_convert or []
55
+ self.full_config = full_config
56
57
if self.weight_bits not in self.TYPE_MAP:
58
raise ValueError(f"Unsupported num_bits = {self.weight_bits}. "
@@ -96,7 +97,7 @@ class AWQMarlinConfig(QuantizationConfig):
97
modules_to_not_convert = cls.get_from_keys_or(
98
config, ["modules_to_not_convert"], None)
99
return cls(weight_bits, group_size, zero_point, lm_head_quantized,
99
- modules_to_not_convert)
100
+ modules_to_not_convert, config)
101
102
@classmethod
103
def override_quantization_method(cls, hf_quant_cfg,
@@ -124,6 +125,13 @@ class AWQMarlinConfig(QuantizationConfig):
125
(isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
126
if is_layer_skipped_awq(prefix, self.modules_to_not_convert):
127
return UnquantizedLinearMethod()
128
+ # Check if the layer is supported by AWQMarlin.
129
+ if not check_marlin_supports_layer(layer, self.group_size):
130
+ logger.warning_once(
131
+ f"Layer '{prefix}' is not supported by AWQMarlin. "
132
+ "Falling back to unoptimized AWQ kernels.")
133
+ return AWQConfig.from_config(
134
+ self.full_config).get_quant_method(layer, prefix)
135
return AWQMarlinLinearMethod(self)
136
elif isinstance(layer, FusedMoE):
137
return AWQMoEMethod(self)
vllm/model_executor/layers/quantization/moe_wna16.py
+6
-3
@@ -16,6 +16,8 @@ from vllm.model_executor.layers.quantization.base_config import (
16
from vllm.model_executor.layers.quantization.gptq import GPTQConfig
17
from vllm.model_executor.layers.quantization.gptq_marlin import (
18
GPTQMarlinConfig)
19
+from vllm.model_executor.layers.quantization.utils.marlin_utils import (
20
+ check_marlin_supports_layer)
21
from vllm.model_executor.utils import set_weight_attrs
22
from vllm.platforms import current_platform
23
@@ -87,8 +89,8 @@ class MoeWNA16Config(QuantizationConfig):
89
modules_to_not_convert = []
90
elif linear_quant_method == "awq":
91
has_zp = cls.get_from_keys(config, ["zero_point"])
90
- modules_to_not_convert = cls.get_from_keys(
91
- config, ["modules_to_not_convert"])
92
+ modules_to_not_convert = cls.get_from_keys_or(
93
+ config, ["modules_to_not_convert"], None)
94
else:
95
raise ValueError("moe_wna16 only support gptq and awq.")
96
@@ -135,7 +137,8 @@ class MoeWNA16Config(QuantizationConfig):
137
return GPTQConfig.from_config(
138
self.full_config).get_quant_method(layer, prefix)
139
elif self.linear_quant_method == "awq":
138
- if self.use_marlin:
140
+ if self.use_marlin and check_marlin_supports_layer(
141
+ layer, self.group_size):
142
return AWQMarlinConfig.from_config(
143
self.full_config).get_quant_method(layer, prefix)
144
else:
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+15
@@ -6,6 +6,7 @@ import numpy
6
import torch
7
8
from vllm import _custom_ops as ops
9
+from vllm.model_executor.layers.linear import LinearBase
10
from vllm.platforms import current_platform
11
from vllm.scalar_type import ScalarType, scalar_types
12
@@ -135,6 +136,20 @@ def check_marlin_supports_shape(output_size_per_partition: int,
136
return True, None
137
138
139
+def check_marlin_supports_layer(layer: LinearBase, group_size: int) \
140
+ -> bool:
141
+ output_size_per_partition = getattr(layer, "output_size_per_partition",
142
+ None) or layer.output_size
143
+ input_size_per_partition = getattr(layer, "input_size_per_partition",
144
+ None) or layer.input_size
145
+
146
+ return check_marlin_supports_shape(
147
+ output_size_per_partition=output_size_per_partition,
148
+ input_size_per_partition=input_size_per_partition,
149
+ input_size=layer.input_size,
150
+ group_size=group_size)[0]
151
+
152
+
153
def marlin_make_workspace(output_size_per_partition: int,
154
device: torch.device) -> torch.Tensor:
155
max_workspace_size = (output_size_per_partition //