A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 02:00 UTC
[ROCm][Quantization] add apply_vllm_mapper in quark config for models like gpt-oss (#28638)
Signed-off-by: xuebwang-amd <xuebwang@amd.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
xuebwang-amd committed
Nov 18, 2025 at 11:16 UTC
d0a73620cc85a840323d25b28772efac04c006e2
1 file changed
+30
-5
vllm/model_executor/layers/quantization/quark/quark.py
+30
-5
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.quantization.quark.utils import (
32
deep_compare,
33
should_ignore_layer,
34
)
35
+from vllm.model_executor.models.utils import WeightsMapper
36
from vllm.platforms import current_platform
37
38
if TYPE_CHECKING:
@@ -57,7 +58,6 @@ class QuarkConfig(QuantizationConfig):
58
self.kv_cache_group = kv_cache_group
59
self.kv_cache_config = kv_cache_config
60
self.pack_method = pack_method
60
- self.ignore: list[str] = cast(list[str], self.quant_config.get("exclude", []))
61
62
def get_linear_method(self) -> "QuarkLinearMethod":
63
return QuarkLinearMethod(self)
@@ -72,14 +72,42 @@ class QuarkConfig(QuantizationConfig):
72
def get_name(self) -> QuantizationMethods:
73
return "quark"
74
75
+ def apply_vllm_mapper( # noqa: B027
76
+ self, hf_to_vllm_mapper: "WeightsMapper"
77
+ ):
78
+ """
79
+ Interface for models to update module names referenced in
80
+ quantization configs in order to reflect the vllm model structure
81
+
82
+ :param hf_to_vllm_mapper: maps from hf model structure (the assumed
83
+ structure of the qconfig) to vllm model structure
84
+ """
85
+ quant_config_with_hf_to_vllm_mapper = {}
86
+
87
+ for k, v in self.quant_config.items():
88
+ if isinstance(v, list):
89
+ quant_config_with_hf_to_vllm_mapper[k] = hf_to_vllm_mapper.apply_list(v)
90
+ elif isinstance(v, dict):
91
+ quant_config_with_hf_to_vllm_mapper[k] = hf_to_vllm_mapper.apply_dict(v)
92
+ else:
93
+ if isinstance(v, str):
94
+ mapped_v_list = hf_to_vllm_mapper.apply_list([v])
95
+ if mapped_v_list:
96
+ quant_config_with_hf_to_vllm_mapper[k] = mapped_v_list[0]
97
+ else:
98
+ quant_config_with_hf_to_vllm_mapper[k] = v
99
+
100
+ self.quant_config = quant_config_with_hf_to_vllm_mapper
101
+
102
def get_quant_method(
103
self, layer: torch.nn.Module, prefix: str
104
) -> Optional["QuantizeMethodBase"]:
105
from vllm.attention.layer import Attention # Avoid circular import
106
107
# Check if the layer is skipped for quantization.
108
+ exclude_layers = cast(list[str], self.quant_config.get("exclude"))
109
if should_ignore_layer(
82
- prefix, ignore=self.ignore, fused_mapping=self.packed_modules_mapping
110
+ prefix, ignore=exclude_layers, fused_mapping=self.packed_modules_mapping
111
):
112
return UnquantizedLinearMethod()
113
if isinstance(layer, LinearBase):
@@ -93,9 +121,6 @@ class QuarkConfig(QuantizationConfig):
121
return QuarkMoEMethod.get_moe_method(self, module=layer, layer_name=prefix)
122
return None
123
96
- def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
97
- self.ignore = hf_to_vllm_mapper.apply_list(self.ignore)
98
-
124
@classmethod
125
def from_config(cls, config: dict[str, Any]) -> "QuarkConfig":
126
export_config = config.get("export")