A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 22:35 UTC
[Fix] Fix llama4 modelopt weight loading error (#22107)
Signed-off-by: jiahanc <173873397+jiahanc@users.noreply.github.com> Co-authored-by: mgoin <mgoin64@gmail.com>
jiahanc committed
Aug 3, 2025 at 00:50 UTC
337eb23bcca6257a75e2c8677c4698bbff9f4a81
1 file changed
+5
-4
vllm/model_executor/models/mllama4.py
+5
-4
@@ -906,11 +906,13 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,
906
def _rename_weight_for_modelopt_checkpoint(self, name: str) -> str:
907
"""Rename weights from ModelOpt llama4 fp8 checkpoints to vLLM
908
format."""
909
- if name.startswith("model."):
909
+ if name.startswith("model.") or name.startswith(
910
+ "language_model.model."):
911
+ renamed = name.replace("model.", "language_model.model.",
912
+ 1) if name.startswith("model.") else name
913
# Handle expert scale parameters with flat naming
914
if "feed_forward.experts." in name and ("_input_scale" in name or
915
"_weight_scale" in name):
913
- renamed = name.replace("model.", "language_model.model.", 1)
916
# Map checkpoint naming to vLLM's expected naming
917
if "down_proj_input_scale" in renamed:
918
return renamed.replace("down_proj_input_scale",
@@ -929,7 +931,6 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,
931
# Handle attention scale parameters
932
elif "self_attn." in name and (".k_scale" in name
933
or ".v_scale" in name):
932
- renamed = name.replace("model.", "language_model.model.", 1)
934
if ".k_proj.k_scale" in renamed:
935
return renamed.replace(".k_proj.k_scale", ".attn.k_scale")
936
elif ".v_proj.v_scale" in renamed:
@@ -937,7 +938,7 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,
938
return renamed
939
940
# Standard model.* to language_model.model.* renaming
940
- return name.replace("model.", "language_model.model.", 1)
941
+ return renamed
942
943
elif name.startswith("lm_head.weight"):
944
return name.replace("lm_head.weight",