A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Use correct key "ignore" for config.json non-quantized layers (#25706)
Signed-off-by: Lee Nau <lnau@nvidia.com>
Lee Nau committed
Sep 29, 2025 at 12:07 UTC
d5ab28511c5fca0294d1b445b670e199f202193b
1 file changed
+6
-2
vllm/model_executor/layers/quantization/modelopt.py
+6
-2
@@ -138,13 +138,15 @@ class ModelOptFp8Config(QuantizationConfig):
138
if not quant_method:
139
raise ValueError("Missing 'quant_algo' in quantization config")
140
kv_cache_quant_method = quant_config.get("kv_cache_quant_algo")
141
+ # "exclude_modules" is the key in the legacy hf_quant_config.json
142
exclude_modules = quant_config.get("exclude_modules")
143
else:
144
# Compressed-tensors style format:
145
# {"quant_algo": "...", "quant_method": "modelopt"}
146
quant_method = config.get("quant_algo", "")
147
kv_cache_quant_method = config.get("kv_cache_quant_algo")
147
- exclude_modules = config.get("exclude_modules")
148
+ # "ignore" is the key in config.json
149
+ exclude_modules = config.get("ignore")
150
151
if quant_method not in QUANT_ALGOS:
152
raise ValueError(
@@ -723,6 +725,7 @@ class ModelOptNvFp4Config(QuantizationConfig):
725
raise ValueError(f"group_size must be an integer, got "
726
f"{type(group_size_raw)}") from None
727
728
+ # "exclude_modules" is the key in the legacy hf_quant_config.json
729
exclude_modules = quant_config.get("exclude_modules", [])
730
if not isinstance(exclude_modules, list):
731
raise ValueError(f"exclude_modules must be a list, got "
@@ -756,7 +759,8 @@ class ModelOptNvFp4Config(QuantizationConfig):
759
raise ValueError(f"group_size must be an integer, got "
760
f"{type(group_size_raw)}") from None
761
759
- exclude_modules = config.get("exclude_modules", [])
762
+ # "ignore" is the key in config.json
763
+ exclude_modules = config.get("ignore", [])
764
if not isinstance(exclude_modules, list):
765
raise ValueError(f"exclude_modules must be a list, got "
766
f"{type(exclude_modules)}")