A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 12:30 UTC
Nvidia ModelOpt workaround for issue 28072 (#30164)
Signed-off-by: Shengliang Xu <shengliangx@nvidia.com> Co-authored-by: Pavani Majety <pmajety@nvidia.com>
Shengliang Xu committed
Dec 14, 2025 at 02:18 UTC
0bb0bae43696d59f8e4d88bd7c6daa992fd31af4
1 file changed
+18
-1
vllm/model_executor/layers/quantization/modelopt.py
+18
-1
@@ -188,7 +188,24 @@ class ModelOptQuantConfigBase(QuantizationConfig):
188
189
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
190
if len(self.exclude_modules) > 0:
191
- self.exclude_modules = hf_to_vllm_mapper.apply_list(self.exclude_modules)
191
+ # This is a workaround for the weights remapping issue:
192
+ # https://github.com/vllm-project/vllm/issues/28072
193
+ # Right now, the Nvidia ModelOpt library use just one wildcard pattern:
194
+ # module_path*
195
+ # It gets applied if the whole tree of modules rooted at module_path
196
+ # is not quantized. Here we replace such pattern by 2 patterns that are
197
+ # collectively equivalent to the original pattern:
198
+ # module_path
199
+ # module_path.*
200
+ new_exclude_modules = []
201
+ for exclude in self.exclude_modules:
202
+ if len(exclude) >= 2 and exclude[-1] == "*" and exclude[-2] != ".":
203
+ new_exclude_modules.append(exclude[:-1])
204
+ new_exclude_modules.append(exclude[:-1] + ".*")
205
+ else:
206
+ new_exclude_modules.append(exclude)
207
+
208
+ self.exclude_modules = hf_to_vllm_mapper.apply_list(new_exclude_modules)
209
210
@staticmethod
211
def get_config_filenames() -> list[str]: