@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b02fd288b2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Hardware][NV] Fix Modelopt model loading for k-v-scales for Llama models. (#11787)

Signed-off-by: Pavani Majety <pmajety@nvidia.com> Co-authored-by: mgoin <michael@neuralmagic.com>

Pavani Majety committed Jan 29, 2025 at 01:46 UTC b02fd288b28f0bfa2d7ac8958fe0d71ec22ffc1b
3 files changed +20 -6
vllm/model_executor/model_loader/weight_utils.py
+10 -1
@@ -652,9 +652,18 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> Optional[str]:
652 return remapped_name
653
654 possible_scale_names = [".k_scale", ".v_scale"]
655 + modelopt_scale_names = [
656 + ".self_attn.k_proj.k_scale", ".self_attn.v_proj.v_scale"
657 + ]
658 for scale_name in possible_scale_names:
659 if name.endswith(scale_name):
657 - remapped_name = name.replace(scale_name, f".attn{scale_name}")
660 + if any(mo_scale_name in name
661 + for mo_scale_name in modelopt_scale_names):
662 + remapped_name = name.replace(
663 + f".self_attn.{scale_name[1]}_proj{scale_name}",
664 + f".self_attn.attn{scale_name}")
665 + else:
666 + remapped_name = name.replace(scale_name, f".attn{scale_name}")
667 if remapped_name not in params_dict:
668 logger.warning_once(
669 f"Found {scale_name} in the checkpoint (e.g. {name}), "
vllm/model_executor/models/llama.py
+5 -4
@@ -404,6 +404,11 @@ class LlamaModel(nn.Module):
404 weight_loader(param, loaded_weight)
405 loaded_params.add(scale_name)
406 continue
407 + if "scale" in name:
408 + # Remapping the name of FP8 kv-scale.
409 + name = maybe_remap_kv_scale_name(name, params_dict)
410 + if name is None:
411 + continue
412 for param_name, weight_name, shard_id in stacked_params_mapping:
413 if weight_name not in name:
414 continue
@@ -423,10 +428,6 @@ class LlamaModel(nn.Module):
428 # Skip loading extra bias for GPTQ models.
429 if name.endswith(".bias") and name not in params_dict:
430 continue
426 - # Remapping the name of FP8 kv-scale.
427 - name = maybe_remap_kv_scale_name(name, params_dict)
428 - if name is None:
429 - continue
431
432 if is_pp_missing_parameter(name, self):
433 continue
vllm/model_executor/models/mixtral.py
+5 -1
@@ -452,7 +452,11 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
452 # Skip layers on other devices.
453 if is_pp_missing_parameter(name, self):
454 continue
455 -
455 + if name.endswith("scale"):
456 + # Remapping the name of FP8 kv-scale.
457 + name = maybe_remap_kv_scale_name(name, params_dict)
458 + if name is None:
459 + continue
460 param = params_dict[name]
461 weight_loader = param.weight_loader
462 weight_loader(param, loaded_weight, shard_id)