A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 04:30 UTC
LLaMA4 LoRA Adapter Enablement (#28602)
Signed-off-by: Fardin Hoque <kfhfar@amazon.com> Co-authored-by: Wei Wei <wwei6@meta.com>
Fardin Hoque committed
Nov 14, 2025 at 10:27 UTC
964d65deedb9ae0480fecdb2e726ba16d63409d7
1 file changed
+34
-2
vllm/model_executor/models/mllama4.py
+34
-2
@@ -35,6 +35,7 @@ from vllm.attention.layer import MultiHeadAttention
35
from vllm.config import VllmConfig
36
from vllm.config.multimodal import BaseDummyOptions
37
from vllm.distributed import get_tensor_model_parallel_world_size
38
+from vllm.model_executor.layers.fused_moe import FusedMoE
39
from vllm.model_executor.layers.linear import (
40
ColumnParallelLinear,
41
QKVParallelLinear,
@@ -45,6 +46,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
46
from vllm.model_executor.layers.rotary_embedding import get_rope
47
from vllm.model_executor.model_loader.utils import initialize_model
48
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
49
+from vllm.model_executor.models.module_mapping import MultiModelKeys
50
from vllm.multimodal import MULTIMODAL_REGISTRY
51
from vllm.multimodal.inputs import (
52
MultiModalDataDict,
@@ -68,11 +70,15 @@ from .interfaces import (
70
MixtureOfExperts,
71
MultiModalEmbeddings,
72
SupportsEagle3,
73
+ SupportsLoRA,
74
SupportsMultiModal,
75
SupportsPP,
76
)
77
from .llama4 import Llama4ForCausalLM
75
-from .utils import AutoWeightsLoader, maybe_prefix
78
+from .utils import (
79
+ AutoWeightsLoader,
80
+ maybe_prefix,
81
+)
82
from .vision import run_dp_sharded_vision_model
83
84
@@ -724,7 +730,12 @@ class Mllama4DummyInputsBuilder(BaseDummyInputsBuilder[Mllama4ProcessingInfo]):
730
dummy_inputs=Mllama4DummyInputsBuilder,
731
)
732
class Llama4ForConditionalGeneration(
727
- nn.Module, SupportsMultiModal, SupportsPP, MixtureOfExperts, SupportsEagle3
733
+ nn.Module,
734
+ SupportsMultiModal,
735
+ SupportsPP,
736
+ MixtureOfExperts,
737
+ SupportsEagle3,
738
+ SupportsLoRA,
739
):
740
merge_by_field_config = True
741
@@ -1067,6 +1078,17 @@ class Llama4ForConditionalGeneration(
1078
1079
return updated_params
1080
1081
+ def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
1082
+ # Params for weights, fp8 weight scales, fp8 activation scales
1083
+ # (param_name, weight_name, expert_id, shard_id)
1084
+ return FusedMoE.make_expert_params_mapping(
1085
+ ckpt_gate_proj_name="gate_proj",
1086
+ ckpt_down_proj_name="down_proj",
1087
+ ckpt_up_proj_name="up_proj",
1088
+ num_experts=self.config.text_config.num_local_experts,
1089
+ num_redundant_experts=self.num_redundant_experts,
1090
+ )
1091
+
1092
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
1093
stacked_params_mapping = [
1094
# (param_name, shard_name, shard_id)
@@ -1113,3 +1135,13 @@ class Llama4ForConditionalGeneration(
1135
)
1136
1137
return updated_params
1138
+
1139
+ def get_mm_mapping(self) -> MultiModelKeys:
1140
+ """
1141
+ Get the module prefix in multimodal models
1142
+ """
1143
+ return MultiModelKeys.from_string_field(
1144
+ language_model="language_model",
1145
+ connector="multi_modal_projector.",
1146
+ tower_model="vision_model.",
1147
+ )