A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 14:30 UTC
Fix infinite recursive search issue in quark.py (#32779)
Signed-off-by: Yanwen Lin <lyw1124278064@gmail.com> Signed-off-by: Xiao Yu <xiao.yu.dc@outlook.com> Signed-off-by: kimheesu <wlskaka4@gmail.com> Co-authored-by: Yanwen Lin <lyw1124278064@gmail.com> Co-authored-by: Kim Hee Su <wlskaka4@gmail.com>
xiao-llm committed
Mar 17, 2026 at 03:19 UTC
24b4272a8ca6a793b80568486060547b5b392433
1 file changed
+11
-4
vllm/model_executor/layers/quantization/quark/quark.py
+11
-4
@@ -467,10 +467,17 @@ class QuarkConfig(QuantizationConfig):
467
layer_name.replace(proj_name, shard_proj_name)
468
for shard_proj_name in shard_proj_names
469
]
470
- shard_configs = [
471
- self._find_matched_config(shard_name, module)
472
- for shard_name in shard_names
473
- ]
470
+
471
+ shard_configs = []
472
+ for shard_name in shard_names:
473
+ if shard_name == layer_name:
474
+ config = cast(
475
+ dict[str, Any], self.quant_config.get("global_quant_config")
476
+ )
477
+ else:
478
+ config = self._find_matched_config(shard_name, module)
479
+ shard_configs.append(config)
480
+
481
if not all(
482
deep_compare(q_config, shard_configs[0]) for q_config in shard_configs
483
):