A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Hardware][NV] Fix Modelopt model loading for k-v-scales for Llama models. (#11787)
Signed-off-by: Pavani Majety <pmajety@nvidia.com> Co-authored-by: mgoin <michael@neuralmagic.com>
Pavani Majety committed
Jan 29, 2025 at 01:46 UTC
b02fd288b28f0bfa2d7ac8958fe0d71ec22ffc1b
3 files changed
+20
-6
vllm/model_executor/model_loader/weight_utils.py
+10
-1
@@ -652,9 +652,18 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> Optional[str]:
652
return remapped_name
653
654
possible_scale_names = [".k_scale", ".v_scale"]
655
+ modelopt_scale_names = [
656
+ ".self_attn.k_proj.k_scale", ".self_attn.v_proj.v_scale"
657
+ ]
658
for scale_name in possible_scale_names:
659
if name.endswith(scale_name):
657
- remapped_name = name.replace(scale_name, f".attn{scale_name}")
660
+ if any(mo_scale_name in name
661
+ for mo_scale_name in modelopt_scale_names):
662
+ remapped_name = name.replace(
663
+ f".self_attn.{scale_name[1]}_proj{scale_name}",
664
+ f".self_attn.attn{scale_name}")
665
+ else:
666
+ remapped_name = name.replace(scale_name, f".attn{scale_name}")
667
if remapped_name not in params_dict:
668
logger.warning_once(
669
f"Found {scale_name} in the checkpoint (e.g. {name}), "
vllm/model_executor/models/llama.py
+5
-4
@@ -404,6 +404,11 @@ class LlamaModel(nn.Module):
404
weight_loader(param, loaded_weight)
405
loaded_params.add(scale_name)
406
continue
407
+ if "scale" in name:
408
+ # Remapping the name of FP8 kv-scale.
409
+ name = maybe_remap_kv_scale_name(name, params_dict)
410
+ if name is None:
411
+ continue
412
for param_name, weight_name, shard_id in stacked_params_mapping:
413
if weight_name not in name:
414
continue
@@ -423,10 +428,6 @@ class LlamaModel(nn.Module):
428
# Skip loading extra bias for GPTQ models.
429
if name.endswith(".bias") and name not in params_dict:
430
continue
426
- # Remapping the name of FP8 kv-scale.
427
- name = maybe_remap_kv_scale_name(name, params_dict)
428
- if name is None:
429
- continue
431
432
if is_pp_missing_parameter(name, self):
433
continue
vllm/model_executor/models/mixtral.py
+5
-1
@@ -452,7 +452,11 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
452
# Skip layers on other devices.
453
if is_pp_missing_parameter(name, self):
454
continue
455
-
455
+ if name.endswith("scale"):
456
+ # Remapping the name of FP8 kv-scale.
457
+ name = maybe_remap_kv_scale_name(name, params_dict)
458
+ if name is None:
459
+ continue
460
param = params_dict[name]
461
weight_loader = param.weight_loader
462
weight_loader(param, loaded_weight, shard_id)