A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 01:15 UTC
Fix quantized Falcon-H1 model loading issues (#32728)
Signed-off-by: Shengliang Xu <shengliangx@nvidia.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Shengliang Xu committed
Feb 2, 2026 at 22:31 UTC
f1cb9b554492bd198ea63cf2233f8599aa850723
1 file changed
+14
-2
vllm/model_executor/models/falcon_h1.py
+14
-2
@@ -35,7 +35,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
35
ParallelLMHead,
36
VocabParallelEmbedding,
37
)
38
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
38
+from vllm.model_executor.model_loader.weight_utils import (
39
+ default_weight_loader,
40
+ maybe_remap_kv_scale_name,
41
+)
42
from vllm.sequence import IntermediateTensors
43
from vllm.transformers_utils.config import set_default_rope_theta
44
@@ -278,6 +281,7 @@ class FalconH1AttentionDecoderLayer(nn.Module):
281
self.scaling,
282
num_kv_heads=self.num_kv_heads,
283
cache_config=cache_config,
284
+ quant_config=quant_config,
285
prefix=f"{prefix}.attn",
286
)
287
self.key_multiplier = config.key_multiplier
@@ -360,7 +364,9 @@ class FalconH1ParallelHybrid(nn.Module):
364
self.attention_in_multiplier = config.attention_in_multiplier
365
self.attn_out_multiplier = config.attention_out_multiplier
366
363
- self.feed_forward = FalconH1MLP(config, prefix=f"{prefix}.feed_forward")
367
+ self.feed_forward = FalconH1MLP(
368
+ config, quant_config=quant_config, prefix=f"{prefix}.feed_forward"
369
+ )
370
371
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
372
self.pre_ff_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -647,6 +653,12 @@ class FalconH1ForCausalLM(
653
if "mamba" in name:
654
name = name.replace("mamba", "mamba.mamba")
655
656
+ if "scale" in name:
657
+ # Remapping the name of kv-scale.
658
+ name = maybe_remap_kv_scale_name(name, params_dict)
659
+ if name is None:
660
+ continue
661
+
662
for param_name, weight_name, shard_id in stacked_params_mapping:
663
if weight_name not in name:
664
continue