A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 14:00 UTC
Enable quantized attention in NemotronH models (#31898)
Signed-off-by: Roi Koren <roik@nvidia.com>
roikoren755 committed
Jan 7, 2026 at 19:37 UTC
bf184a66218beae5355c1e9784074c21b896fe0f
2 files changed
+4
vllm/model_executor/model_loader/weight_utils.py
+3
@@ -1153,6 +1153,9 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None:
1153
# Qwen3 MoE format: .self_attn.qkqkv_proj.{k,v}_scale ->
1154
# .self_attn.attn.{k,v}_scale
1155
(r"\.self_attn\.qkqkv_proj\.([kv])_scale$", r".self_attn.attn.\1_scale"),
1156
+ # NemotronH format: .mixer.{k,v}_proj.{k,v}_scale ->
1157
+ # .mixer.attn.{k,v}_scale
1158
+ (r"\.mixer\.[kv]_proj\.([kv])_scale$", r".mixer.attn.\1_scale"),
1159
# Default format: .{k,v}_scale -> .attn.{k,v}_scale
1160
(r"\.([kv])_scale$", r".attn.\1_scale"),
1161
]
vllm/model_executor/models/nemotron_h.py
+1
@@ -483,6 +483,7 @@ class NemotronHAttention(nn.Module):
483
self.scaling,
484
num_kv_heads=self.num_kv_heads,
485
cache_config=cache_config,
486
+ quant_config=quant_config,
487
prefix=f"{prefix}.attn",
488
)
489