A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 17:30 UTC
[bugfix][quantization] fix quark qwen3 kv_cache quantization (#30308)
Signed-off-by: Haoyang Li <lihaoyang0109@gmail.com>
haoyangli-amd committed
Dec 10, 2025 at 11:24 UTC
06462392e40f9ae1bf87290c4cec10533fdd3205
1 file changed
+14
vllm/model_executor/models/qwen3_moe.py
+14
@@ -403,6 +403,7 @@ class Qwen3MoeModel(nn.Module):
403
self.padding_idx = config.pad_token_id
404
self.vocab_size = config.vocab_size
405
self.config = config
406
+ self.quant_config = quant_config
407
self.embed_tokens = VocabParallelEmbedding(
408
config.vocab_size,
409
config.hidden_size,
@@ -505,6 +506,19 @@ class Qwen3MoeModel(nn.Module):
506
loaded_params: set[str] = set()
507
expert_params_mapping = self.get_expert_mapping()
508
for name, loaded_weight in weights:
509
+ if self.quant_config is not None and (
510
+ scale_name := self.quant_config.get_cache_scale(name)
511
+ ):
512
+ # Loading kv cache quantization scales
513
+ param = params_dict[scale_name]
514
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
515
+ assert loaded_weight.numel() == 1, (
516
+ f"KV scale numel {loaded_weight.numel()} != 1"
517
+ )
518
+ loaded_weight = loaded_weight.squeeze()
519
+ weight_loader(param, loaded_weight)
520
+ loaded_params.add(scale_name)
521
+ continue
522
for param_name, weight_name, shard_id in stacked_params_mapping:
523
# Skip non-stacked layers and experts (experts handled below).
524
if weight_name not in name: