A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 05:45 UTC
[Misc] add ignore mapper for quark quantization (#28275)
Signed-off-by: Haoyang Li <lihaoyang0109@gmail.com>
haoyangli-amd committed
Nov 14, 2025 at 13:56 UTC
0b25498990f01ea2553c02731d6e2ce2d550156a
1 file changed
+9
-3
vllm/model_executor/layers/quantization/quark/quark.py
+9
-3
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import fnmatch
5
-from typing import Any, Optional, cast
5
+from typing import TYPE_CHECKING, Any, Optional, cast
6
7
import torch
8
@@ -34,6 +34,9 @@ from vllm.model_executor.layers.quantization.quark.utils import (
34
)
35
from vllm.platforms import current_platform
36
37
+if TYPE_CHECKING:
38
+ from vllm.model_executor.models.utils import WeightsMapper
39
+
40
__all__ = ["QuarkLinearMethod"]
41
42
logger = init_logger(__name__)
@@ -54,6 +57,7 @@ class QuarkConfig(QuantizationConfig):
57
self.kv_cache_group = kv_cache_group
58
self.kv_cache_config = kv_cache_config
59
self.pack_method = pack_method
60
+ self.ignore: list[str] = cast(list[str], self.quant_config.get("exclude", []))
61
62
def get_linear_method(self) -> "QuarkLinearMethod":
63
return QuarkLinearMethod(self)
@@ -74,9 +78,8 @@ class QuarkConfig(QuantizationConfig):
78
from vllm.attention.layer import Attention # Avoid circular import
79
80
# Check if the layer is skipped for quantization.
77
- exclude_layers = cast(list[str], self.quant_config.get("exclude"))
81
if should_ignore_layer(
79
- prefix, ignore=exclude_layers, fused_mapping=self.packed_modules_mapping
82
+ prefix, ignore=self.ignore, fused_mapping=self.packed_modules_mapping
83
):
84
return UnquantizedLinearMethod()
85
if isinstance(layer, LinearBase):
@@ -90,6 +93,9 @@ class QuarkConfig(QuantizationConfig):
93
return QuarkMoEMethod.get_moe_method(self, module=layer, layer_name=prefix)
94
return None
95
96
+ def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
97
+ self.ignore = hf_to_vllm_mapper.apply_list(self.ignore)
98
+
99
@classmethod
100
def from_config(cls, config: dict[str, Any]) -> "QuarkConfig":
101
export_config = config.get("export")