A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Bugfix] fix pp for llama4 (#16746)
Signed-off-by: Lu Fang <fanglu@fb.com>
Lucia Fang committed
Apr 17, 2025 at 22:51 UTC
e31045f95ca0f7262b156cd7e3e34100cbf1f4d1
1 file changed
+3
-3
vllm/model_executor/models/mllama4.py
+3
-3
@@ -672,9 +672,9 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,
672
self.config,
673
None,
674
prefix=maybe_prefix(prefix, "multi_modal_projector"))
675
-
675
self.language_model = _initialize_model(
677
- vllm_config=vllm_config.with_hf_config(config.text_config),
676
+ vllm_config=vllm_config.with_hf_config(config.text_config,
677
+ ["LlamaForCausalLM"]),
678
prefix=maybe_prefix(prefix, "language_model"),
679
model_class=Llama4ForCausalLM,
680
)
@@ -824,7 +824,7 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,
824
# language_model is an Llama4ForCausalLM instance. We load it's
825
# using llama4's load_weights routine.
826
language_model_weights, other_weights = self.separate_weights(
827
- weights, prefix="language_model.model.")
827
+ weights, prefix="language_model.")
828
loader = AutoWeightsLoader(self)
829
loaded_language_model_params = loader.load_weights(
830
language_model_weights)