@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 352c0c8a28

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Quantization] Automatically infer AWQ `modules_to_not_convert` field (#26909)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>

Isotr0py committed Oct 21, 2025 at 09:49 UTC 352c0c8a285414b11373e65fef095af7b07b94d8

6 files changed +96 -45

vllm/model_executor/layers/quantization/awq.py

+34 -6

@@ -1,9 +1,10 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 -from typing import Any, Union

4 +from typing import TYPE_CHECKING, Any, Union

6 import torch

7 +from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE

9 from vllm import _custom_ops as ops

10 from vllm.logger import init_logger

@@ -13,12 +14,17 @@ from vllm.model_executor.layers.linear import (

14 LinearMethodBase,

15 UnquantizedLinearMethod,

16 )

16 -from vllm.model_executor.layers.quantization import QuantizationMethods

17 from vllm.model_executor.layers.quantization.base_config import (

18 QuantizationConfig,

19 QuantizeMethodBase,

20 )

21 +from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped

22 from vllm.model_executor.parameter import GroupQuantScaleParameter, PackedvLLMParameter

23 +from vllm.transformers_utils.config import get_safetensors_params_metadata

24 +

25 +if TYPE_CHECKING:

26 + from vllm.model_executor.layers.quantization import QuantizationMethods

27 + from vllm.model_executor.models.utils import WeightsMapper

29 logger = init_logger(__name__)

@@ -57,7 +63,7 @@ class AWQConfig(QuantizationConfig):

63 f"modules_to_not_convert={self.modules_to_not_convert})"

64 )

60 - def get_name(self) -> QuantizationMethods:

66 + def get_name(self) -> "QuantizationMethods":

67 return "awq"

69 def get_supported_act_dtypes(self) -> list[torch.dtype]:

@@ -90,7 +96,12 @@ class AWQConfig(QuantizationConfig):

96 self, layer: torch.nn.Module, prefix: str

97 ) -> Union["LinearMethodBase", "QuantizeMethodBase"] | None:

98 if isinstance(layer, LinearBase):

93 - if is_layer_skipped_awq(prefix, self.modules_to_not_convert):

99 + if is_layer_skipped(

100 + prefix,

101 + self.modules_to_not_convert,

102 + self.packed_modules_mapping,

103 + skip_with_substr=True,

104 + ):

105 return UnquantizedLinearMethod()

106 return AWQLinearMethod(self)

107 elif isinstance(layer, FusedMoE):

@@ -128,9 +139,26 @@ class AWQConfig(QuantizationConfig):

139 return AWQMoEMethod(awq_marlin_config, layer.moe_config)

140 return None

141

142 + def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):

143 + if self.modules_to_not_convert:

144 + self.modules_to_not_convert = hf_to_vllm_mapper.apply_list(

145 + self.modules_to_not_convert

146 + )

147

132 -def is_layer_skipped_awq(prefix: str, modules_to_not_convert: list[str]):

133 - return any(module_name in prefix for module_name in modules_to_not_convert)

148 + def maybe_update_config(self, model_name: str, revision: str | None = None):

149 + if self.modules_to_not_convert:

150 + return

151 +

152 + unquant_dtypes = [torch.float16, torch.bfloat16, torch.float32]

153 + metadata = get_safetensors_params_metadata(model_name, revision=revision)

154 + layers = {param_name.rsplit(".", 1)[0] for param_name in metadata}

155 + quant_layers: set[str] = {

156 + param_name.rsplit(".", 1)[0]

157 + for param_name, info in metadata.items()

158 + if (dtype := info.get("dtype", None))

159 + and _SAFETENSORS_TO_TORCH_DTYPE[dtype] not in unquant_dtypes

160 + }

161 + self.modules_to_not_convert = list(layers - quant_layers)

162

163

164 class AWQLinearMethod(LinearMethodBase):

vllm/model_executor/layers/quantization/awq_marlin.py

+36 -9

@@ -2,9 +2,10 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 from collections.abc import Callable

5 -from typing import Any, Optional

5 +from typing import TYPE_CHECKING, Any, Optional

7 import torch

8 +from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE

9 from torch.nn import Parameter

11 import vllm.model_executor.layers.fused_moe # noqa

@@ -27,8 +28,7 @@ from vllm.model_executor.layers.linear import (

28 UnquantizedLinearMethod,

29 set_weight_attrs,

30 )

30 -from vllm.model_executor.layers.quantization import QuantizationMethods

31 -from vllm.model_executor.layers.quantization.awq import AWQConfig, is_layer_skipped_awq

31 +from vllm.model_executor.layers.quantization.awq import AWQConfig

32 from vllm.model_executor.layers.quantization.base_config import (

33 QuantizationConfig,

34 QuantizeMethodBase,

@@ -49,10 +49,16 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (

49 verify_marlin_supported,

50 verify_marlin_supports_shape,

51 )

52 +from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped

53 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead

54 from vllm.model_executor.parameter import GroupQuantScaleParameter, PackedvLLMParameter

55 from vllm.platforms import current_platform

56 from vllm.scalar_type import scalar_types

57 +from vllm.transformers_utils.config import get_safetensors_params_metadata

58 +

59 +if TYPE_CHECKING:

60 + from vllm.model_executor.layers.quantization import QuantizationMethods

61 + from vllm.model_executor.models.utils import WeightsMapper

63 logger = init_logger(__name__)

@@ -106,7 +112,7 @@ class AWQMarlinConfig(QuantizationConfig):

112 )

113

114 @classmethod

109 - def get_name(cls) -> QuantizationMethods:

115 + def get_name(cls) -> "QuantizationMethods":

116 return "awq_marlin"

117

118 @classmethod

@@ -142,7 +148,7 @@ class AWQMarlinConfig(QuantizationConfig):

148 @classmethod

149 def override_quantization_method(

150 cls, hf_quant_cfg, user_quant

145 - ) -> QuantizationMethods | None:

151 + ) -> Optional["QuantizationMethods"]:

152 can_convert = cls.is_awq_marlin_compatible(hf_quant_cfg)

153 is_valid_user_quant = (

154 user_quant is None or user_quant == "marlin" or user_quant == "awq_marlin"

@@ -171,7 +177,9 @@ class AWQMarlinConfig(QuantizationConfig):

177 if isinstance(layer, LinearBase) or (

178 isinstance(layer, ParallelLMHead) and self.lm_head_quantized

179 ):

174 - if is_layer_skipped_awq(prefix, self.modules_to_not_convert):

180 + if is_layer_skipped(

181 + prefix, self.modules_to_not_convert, self.packed_modules_mapping

182 + ):

183 return UnquantizedLinearMethod()

184 # Check if the layer is supported by AWQMarlin.

185 if not check_marlin_supports_layer(layer, self.group_size):

@@ -186,9 +194,7 @@ class AWQMarlinConfig(QuantizationConfig):

194 elif isinstance(layer, FusedMoE):

195 from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config

196

189 - if is_layer_skipped_awq(

190 - prefix, getattr(self, "modules_to_not_convert", [])

191 - ):

197 + if is_layer_skipped(prefix, getattr(self, "modules_to_not_convert", [])):

198 return UnquantizedFusedMoEMethod(layer.moe_config)

199 if not check_moe_marlin_supports_layer(layer, self.group_size):

200 logger.warning_once(

@@ -226,6 +232,27 @@ class AWQMarlinConfig(QuantizationConfig):

232 quant_type=cls.TYPE_MAP[num_bits], group_size=group_size, has_zp=zero_point

233 )

234

235 + def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):

236 + if self.modules_to_not_convert:

237 + self.modules_to_not_convert = hf_to_vllm_mapper.apply_list(

238 + self.modules_to_not_convert

239 + )

240 +

241 + def maybe_update_config(self, model_name: str, revision: str | None = None):

242 + if self.modules_to_not_convert:

243 + return

244 +

245 + unquant_dtypes = [torch.float16, torch.bfloat16, torch.float32]

246 + metadata = get_safetensors_params_metadata(model_name, revision=revision)

247 + layers = {param_name.rsplit(".", 1)[0] for param_name in metadata}

248 + quant_layers: set[str] = {

249 + param_name.rsplit(".", 1)[0]

250 + for param_name, info in metadata.items()

251 + if (dtype := info.get("dtype", None))

252 + and _SAFETENSORS_TO_TORCH_DTYPE[dtype] not in unquant_dtypes

253 + }

254 + self.modules_to_not_convert = list(layers - quant_layers)

255 +

256

257 class AWQMarlinLinearMethod(LinearMethodBase):

258 """Linear method for AWQ Marlin.

vllm/model_executor/layers/quantization/gptq.py

+2 -1

@@ -32,6 +32,7 @@ from vllm.utils.collection_utils import is_list_of

33 if TYPE_CHECKING:

34 from vllm.model_executor.layers.quantization import QuantizationMethods

35 + from vllm.model_executor.models.utils import WeightsMapper

36 else:

37 QuantizationMethods = str

@@ -164,7 +165,7 @@ class GPTQConfig(QuantizationConfig):

165

166 return get_linear_quant_method(self, layer, prefix, GPTQLinearMethod)

167

167 - def apply_vllm_mapper(self, hf_to_vllm_mapper):

168 + def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):

169 if self.modules_in_block_to_quantize is not None:

170 self.modules_in_block_to_quantize = hf_to_vllm_mapper.apply_list(

171 self.modules_in_block_to_quantize

vllm/model_executor/layers/quantization/ipex_quant.py

+5 -5

@@ -24,12 +24,10 @@ from vllm.model_executor.layers.quantization import (

24 QuantizationConfig,

25 QuantizationMethods,

26 )

27 -from vllm.model_executor.layers.quantization.awq import (

28 - AWQLinearMethod,

29 - is_layer_skipped_awq,

30 -)

27 +from vllm.model_executor.layers.quantization.awq import AWQLinearMethod

28 from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod

29 from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod

30 +from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped

31 from vllm.model_executor.utils import set_weight_attrs

32 from vllm.platforms import current_platform

@@ -139,7 +137,9 @@ class IPEXConfig(QuantizationConfig):

137 ) -> Optional["LinearMethodBase"]:

138 if isinstance(layer, LinearBase):

139 if self.method == "awq":

142 - if is_layer_skipped_awq(prefix, self.modules_to_not_convert):

140 + if is_layer_skipped(

141 + prefix, self.modules_to_not_convert, self.packed_modules_mapping

142 + ):

143 return UnquantizedLinearMethod()

144 return IPEXAWQLinearMethod(self)

145 if self.method == "gptq":

vllm/model_executor/layers/quantization/utils/quant_utils.py

+19 -8

@@ -285,7 +285,18 @@ def is_layer_skipped(

285 prefix: str,

286 ignored_layers: list[str],

287 fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),

288 + *,

289 + skip_with_substr: bool = False,

290 ) -> bool:

291 + def prefix_full_match(prefix: str, ignored_layers: list[str]) -> bool:

292 + return prefix in ignored_layers

293 +

294 + # For case like: ignored_layers = ["self_attn"]

295 + def substr_match(prefix: str, ignored_layers: list[str]) -> bool:

296 + return any(layer in prefix for layer in ignored_layers)

297 +

298 + match_func = substr_match if skip_with_substr else prefix_full_match

299 +

300 # prefix: model.layers.0.self_attn.q_proj

301 # proj_name: q_proj

302 proj_name = prefix.split(".")[-1]

@@ -302,7 +313,7 @@ def is_layer_skipped(

313

314 is_skipped = None

315 for shard_prefix in shard_prefixes:

305 - is_shard_skipped = shard_prefix in ignored_layers

316 + is_shard_skipped = match_func(shard_prefix, ignored_layers)

317

318 if is_skipped is None:

319 is_skipped = is_shard_skipped

@@ -312,16 +323,16 @@ def is_layer_skipped(

323 "are quantized. All shards of fused layers "

324 "to have the same precision."

325 )

315 - elif "experts" in prefix:

326 + elif "experts" in prefix and not skip_with_substr:

327 + expert_ignore_layers = filter(

328 + lambda layer_name: "experts" in layer_name, ignored_layers

329 + )

330 return any(

317 - [

318 - prefix in layer_name

319 - for layer_name in ignored_layers

320 - if "experts" in layer_name

321 - ]

331 + prefix in layer_name if not skip_with_substr else layer_name in prefix

332 + for layer_name in expert_ignore_layers

333 )

334 else:

324 - is_skipped = prefix in ignored_layers

335 + is_skipped = match_func(prefix, ignored_layers)

336

337 assert is_skipped is not None

338 return is_skipped

vllm/model_executor/models/minicpmv.py

-16

@@ -42,8 +42,6 @@ from typing_extensions import TypeVar

42 from vllm.config import VllmConfig

43 from vllm.config.multimodal import BaseDummyOptions

44 from vllm.model_executor.layers.quantization import QuantizationConfig

45 -from vllm.model_executor.layers.quantization.awq import AWQConfig

46 -from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig

45 from vllm.model_executor.layers.resampler import (

46 BaseResampler,

47 Resampler2,

@@ -1514,11 +1512,6 @@ class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA):

1512 super().__init__(vllm_config=vllm_config, prefix=prefix)

1513 assert self.version == (4, 0)

1514

1517 - def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):

1518 - if isinstance(quant_config, (AWQConfig, AWQMarlinConfig)):

1519 - return None

1520 - return quant_config

1521 -

1515 def init_llm(

1516 self,

1517 vllm_config: VllmConfig,

@@ -1532,7 +1525,6 @@ class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA):

1525 quant_config: QuantizationConfig | None = None,

1526 prefix: str = "",

1527 ) -> nn.Module:

1535 - quant_config = self._maybe_ignore_quant_config(quant_config)

1528 model = Idefics2VisionTransformer(

1529 config.vision_config,

1530 quant_config=quant_config,

@@ -1550,7 +1542,6 @@ class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA):

1542 quant_config: QuantizationConfig | None = None,

1543 prefix: str = "",

1544 ) -> nn.Module:

1553 - quant_config = self._maybe_ignore_quant_config(quant_config)

1545 with set_default_torch_dtype(torch.float16):

1546 # The resampler in 4.0 remains consistent with the one in 2.5/2.6.

1547 resampler = Resampler2_5(

@@ -1619,11 +1610,6 @@ class MiniCPMV4_5(MiniCPMVBaseModel, SupportsLoRA):

1610 super().__init__(vllm_config=vllm_config, prefix=prefix)

1611 assert self.version == (4, 5)

1612

1622 - def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):

1623 - if isinstance(quant_config, (AWQConfig, AWQMarlinConfig)):

1624 - return None

1625 - return quant_config

1626 -

1613 def init_llm(

1614 self,

1615 vllm_config: VllmConfig,

@@ -1637,7 +1623,6 @@ class MiniCPMV4_5(MiniCPMVBaseModel, SupportsLoRA):

1623 quant_config: QuantizationConfig | None = None,

1624 prefix: str = "",

1625 ) -> nn.Module:

1640 - quant_config = self._maybe_ignore_quant_config(quant_config)

1626 model = Idefics2VisionTransformer(

1627 config.vision_config,

1628 quant_config=quant_config,

@@ -1655,7 +1640,6 @@ class MiniCPMV4_5(MiniCPMVBaseModel, SupportsLoRA):

1640 quant_config: QuantizationConfig | None = None,

1641 prefix: str = "",

1642 ) -> nn.Module:

1658 - quant_config = self._maybe_ignore_quant_config(quant_config)

1643 with set_default_torch_dtype(torch.float16):

1644 # The resampler in 4.0 remains consistent with the one in 2.5/2.6.

1645 resampler = Resampler4_5(