A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 20:00 UTC
[Bugfix] Fix compressed-tensors models failing to load with transformers backend (#30287)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Michael Goin committed
Dec 8, 2025 at 23:44 UTC
03b91f726214b4a022f73a084ee283001e5bba0c
1 file changed
+30
-9
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+30
-9
@@ -116,16 +116,37 @@ class CompressedTensorsConfig(QuantizationConfig):
116
return "compressed-tensors"
117
118
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
119
- self.target_scheme_map = hf_to_vllm_mapper.apply_dict(self.target_scheme_map)
120
- self.ignore = hf_to_vllm_mapper.apply_list(self.ignore)
121
- self.sparsity_scheme_map = hf_to_vllm_mapper.apply_dict(
122
- self.sparsity_scheme_map
123
- )
124
- self.sparsity_ignore_list = hf_to_vllm_mapper.apply_list(
125
- self.sparsity_ignore_list
126
- )
119
+ """
120
+ Transform layer paths in config targets to match vLLM's naming.
121
+
122
+ The WeightsMapper is designed for weight paths, but some backends
123
+ (e.g. transformers) use broad prefix mappings like "" -> "model."
124
+ which would incorrectly transform non-path targets.
125
+
126
+ compressed-tensors targets can be:
127
+ - Layer paths: "layers.0.self_attn.q_proj" -> transformed
128
+ - Module class names: "Linear" -> preserved (no ".")
129
+ - Regex patterns: "re:.*proj" -> preserved (starts with "re:")
130
+ """
131
+
132
+ def _map_target(target: str) -> str | None:
133
+ is_layer_path = "." in target and not target.startswith("re:")
134
+ if is_layer_path:
135
+ return hf_to_vllm_mapper._map_name(target)
136
+ return target
137
+
138
+ def _apply_dict(d: dict) -> dict:
139
+ return {k: v for t, v in d.items() if (k := _map_target(t)) is not None}
140
+
141
+ def _apply_list(lst: list) -> list:
142
+ return [t for x in lst if (t := _map_target(x)) is not None]
143
+
144
+ self.target_scheme_map = _apply_dict(self.target_scheme_map)
145
+ self.ignore = _apply_list(self.ignore)
146
+ self.sparsity_scheme_map = _apply_dict(self.sparsity_scheme_map)
147
+ self.sparsity_ignore_list = _apply_list(self.sparsity_ignore_list)
148
if self.kv_cache_scheme is not None:
128
- self.kv_cache_scheme = hf_to_vllm_mapper.apply_dict(self.kv_cache_scheme)
149
+ self.kv_cache_scheme = _apply_dict(self.kv_cache_scheme)
150
151
def get_quant_method(
152
self,